2023-08-10 06:30:18,949 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7c7b252e 2023-08-10 06:30:18,962 main DEBUG Took 0.010554 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-08-10 06:30:18,962 main DEBUG PluginManager 'Core' found 129 plugins 2023-08-10 06:30:18,963 main DEBUG PluginManager 'Level' found 0 plugins 2023-08-10 06:30:18,963 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-08-10 06:30:18,965 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:30:18,974 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-08-10 06:30:18,986 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:30:18,988 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:30:18,988 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:30:18,988 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:30:18,989 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:30:18,989 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:30:18,990 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:30:18,990 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:30:18,990 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:30:18,991 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:30:18,991 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:30:18,991 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:30:18,992 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:30:18,992 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:30:18,992 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:30:18,993 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:30:18,993 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:30:18,993 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:30:18,994 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:30:18,994 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:30:18,994 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:30:18,994 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:30:18,995 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:30:18,995 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-08-10 06:30:18,995 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:30:18,996 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-08-10 06:30:18,998 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-08-10 06:30:18,999 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-08-10 06:30:19,001 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-08-10 06:30:19,001 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-08-10 06:30:19,002 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-08-10 06:30:19,003 main DEBUG PluginManager 'Converter' found 47 plugins 2023-08-10 06:30:19,014 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-08-10 06:30:19,017 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-08-10 06:30:19,018 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-08-10 06:30:19,018 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-08-10 06:30:19,019 main DEBUG createAppenders(={Console}) 2023-08-10 06:30:19,019 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7c7b252e initialized 2023-08-10 06:30:19,020 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7c7b252e 2023-08-10 06:30:19,020 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7c7b252e OK. 2023-08-10 06:30:19,020 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-08-10 06:30:19,021 main DEBUG OutputStream closed 2023-08-10 06:30:19,021 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-08-10 06:30:19,021 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-08-10 06:30:19,021 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@72a7c7e0 OK 2023-08-10 06:30:19,064 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-08-10 06:30:19,065 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-08-10 06:30:19,066 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-08-10 06:30:19,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-08-10 06:30:19,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-08-10 06:30:19,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-08-10 06:30:19,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-08-10 06:30:19,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-08-10 06:30:19,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-08-10 06:30:19,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-08-10 06:30:19,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-08-10 06:30:19,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-08-10 06:30:19,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-08-10 06:30:19,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-08-10 06:30:19,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-08-10 06:30:19,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-08-10 06:30:19,072 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-08-10 06:30:19,073 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-08-10 06:30:19,075 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-08-10 06:30:19,076 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@e7edb54) with optional ClassLoader: null 2023-08-10 06:30:19,076 main DEBUG Shutdown hook enabled. Registering a new one. 2023-08-10 06:30:19,077 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@e7edb54] started OK. 2023-08-10T06:30:19,109 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures timeout: 13 mins 2023-08-10 06:30:19,112 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-08-10 06:30:19,113 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-08-10T06:30:19,419 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2 2023-08-10T06:30:19,424 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-08-10T06:30:19,426 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:30:19,427 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-08-10T06:30:19,431 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/cluster_336bfe0c-247d-9555-0740-69ce104bc2cc, deleteOnExit=true 2023-08-10T06:30:19,431 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-08-10T06:30:19,431 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/test.cache.data in system properties and HBase conf 2023-08-10T06:30:19,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/hadoop.tmp.dir in system properties and HBase conf 2023-08-10T06:30:19,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/hadoop.log.dir in system properties and HBase conf 2023-08-10T06:30:19,432 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/mapreduce.cluster.local.dir in system properties and HBase conf 2023-08-10T06:30:19,433 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-08-10T06:30:19,433 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-08-10T06:30:19,526 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-08-10T06:30:19,942 DEBUG [Time-limited test {}] fs.HFileSystem(308): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-08-10T06:30:19,947 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-08-10T06:30:19,947 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-08-10T06:30:19,948 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-08-10T06:30:19,948 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-08-10T06:30:19,948 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-08-10T06:30:19,949 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-08-10T06:30:19,949 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-08-10T06:30:19,949 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/dfs.journalnode.edits.dir in system properties and HBase conf 2023-08-10T06:30:19,949 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-08-10T06:30:19,950 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/nfs.dump.dir in system properties and HBase conf 2023-08-10T06:30:19,950 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/java.io.tmpdir in system properties and HBase conf 2023-08-10T06:30:19,950 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/dfs.journalnode.edits.dir in system properties and HBase conf 2023-08-10T06:30:19,951 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-08-10T06:30:19,951 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-08-10T06:30:20,400 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-08-10T06:30:20,404 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-08-10T06:30:20,680 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-08-10T06:30:20,816 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@901089d via org.mortbay.log.Slf4jLog 2023-08-10T06:30:20,832 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:30:20,875 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:30:20,913 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/java.io.tmpdir/Jetty_localhost_38375_hdfs____i7o6t2/webapp 2023-08-10T06:30:21,048 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:38375 2023-08-10T06:30:21,076 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-08-10T06:30:21,076 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-08-10T06:30:21,638 WARN [Listener at localhost/43579 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-08-10T06:30:21,704 WARN [Listener at localhost/43579 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-08-10T06:30:21,721 WARN [Listener at localhost/43579 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:30:21,727 INFO [Listener at localhost/43579 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:30:21,733 INFO [Listener at localhost/43579 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/java.io.tmpdir/Jetty_localhost_35187_datanode____eoiw51/webapp 2023-08-10T06:30:21,830 INFO [Listener at localhost/43579 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35187 2023-08-10T06:30:22,120 WARN [Listener at localhost/39735 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-08-10T06:30:22,387 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-08-10T06:30:22,551 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xf9f54961bea029a2: Processing first storage report for DS-f50d05d4-03fd-404d-8664-f53b0b5949ce from datanode 08fffa40-adf7-4a28-ba0e-d7b0211e926d 2023-08-10T06:30:22,553 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xf9f54961bea029a2: from storage DS-f50d05d4-03fd-404d-8664-f53b0b5949ce node DatanodeRegistration(127.0.0.1:46635, datanodeUuid=08fffa40-adf7-4a28-ba0e-d7b0211e926d, infoPort=36981, infoSecurePort=0, ipcPort=39735, storageInfo=lv=-57;cid=testClusterID;nsid=909538877;c=1691649020475), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2023-08-10T06:30:22,553 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0xf9f54961bea029a2: Processing first storage report for DS-cd036fff-6275-41e4-85d0-3d5660b4caad from datanode 08fffa40-adf7-4a28-ba0e-d7b0211e926d 2023-08-10T06:30:22,553 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0xf9f54961bea029a2: from storage DS-cd036fff-6275-41e4-85d0-3d5660b4caad node DatanodeRegistration(127.0.0.1:46635, datanodeUuid=08fffa40-adf7-4a28-ba0e-d7b0211e926d, infoPort=36981, infoSecurePort=0, ipcPort=39735, storageInfo=lv=-57;cid=testClusterID;nsid=909538877;c=1691649020475), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-08-10T06:30:22,641 DEBUG [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2 2023-08-10T06:30:22,726 INFO [Listener at localhost/39735 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/cluster_336bfe0c-247d-9555-0740-69ce104bc2cc/zookeeper_0, clientPort=55224, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/cluster_336bfe0c-247d-9555-0740-69ce104bc2cc/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/cluster_336bfe0c-247d-9555-0740-69ce104bc2cc/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-08-10T06:30:22,744 INFO [Listener at localhost/39735 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=55224 2023-08-10T06:30:22,767 INFO [Listener at localhost/39735 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:30:22,769 INFO [Listener at localhost/39735 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:30:23,427 INFO [Listener at localhost/39735 {}] util.FSUtils(461): Created version file at hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a with version=8 2023-08-10T06:30:23,427 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/hbase-staging 2023-08-10T06:30:23,445 DEBUG [Listener at localhost/39735 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-08-10T06:30:23,445 DEBUG [Listener at localhost/39735 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-08-10T06:30:23,445 DEBUG [Listener at localhost/39735 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-08-10T06:30:23,445 DEBUG [Listener at localhost/39735 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-08-10T06:30:23,661 DEBUG [Listener at localhost/39735 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-08-10T06:30:23,817 INFO [Listener at localhost/39735 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-08-10T06:30:24,448 INFO [Listener at localhost/39735 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-08-10T06:30:24,494 INFO [Listener at localhost/39735 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-08-10T06:30:24,495 INFO [Listener at localhost/39735 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-08-10T06:30:24,495 INFO [Listener at localhost/39735 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-08-10T06:30:24,495 INFO [Listener at localhost/39735 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-08-10T06:30:24,495 INFO [Listener at localhost/39735 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-08-10T06:30:24,646 INFO [Listener at localhost/39735 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-08-10T06:30:24,753 DEBUG [Listener at localhost/39735 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-08-10T06:30:24,759 INFO [Listener at localhost/39735 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-08-10T06:30:24,809 DEBUG [Listener at localhost/39735 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 16879 (auto-detected) 2023-08-10T06:30:24,811 DEBUG [Listener at localhost/39735 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-08-10T06:30:24,848 INFO [Listener at localhost/39735 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:46449 2023-08-10T06:30:24,862 INFO [Listener at localhost/39735 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:30:24,865 INFO [Listener at localhost/39735 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:30:24,884 INFO [Listener at localhost/39735 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:46449 connecting to ZooKeeper ensemble=127.0.0.1:55224 2023-08-10T06:30:24,931 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:464490x0, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-08-10T06:30:24,934 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:46449-0x10060ecfb4e0000 connected 2023-08-10T06:30:24,961 DEBUG [Listener at localhost/39735 {}] zookeeper.ZKUtil(113): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-08-10T06:30:24,964 DEBUG [Listener at localhost/39735 {}] zookeeper.ZKUtil(113): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-08-10T06:30:24,968 DEBUG [Listener at localhost/39735 {}] zookeeper.ZKUtil(113): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-08-10T06:30:24,978 DEBUG [Listener at localhost/39735 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46449 2023-08-10T06:30:24,979 DEBUG [Listener at localhost/39735 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46449 2023-08-10T06:30:24,979 DEBUG [Listener at localhost/39735 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46449 2023-08-10T06:30:24,980 DEBUG [Listener at localhost/39735 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46449 2023-08-10T06:30:24,980 DEBUG [Listener at localhost/39735 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46449 2023-08-10T06:30:25,034 INFO [Listener at localhost/39735 {}] log.Log(170): Logging initialized @6722ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-08-10T06:30:25,154 INFO [Listener at localhost/39735 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-08-10T06:30:25,155 INFO [Listener at localhost/39735 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-08-10T06:30:25,156 INFO [Listener at localhost/39735 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-08-10T06:30:25,158 INFO [Listener at localhost/39735 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-08-10T06:30:25,158 INFO [Listener at localhost/39735 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-08-10T06:30:25,158 INFO [Listener at localhost/39735 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-08-10T06:30:25,162 INFO [Listener at localhost/39735 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-08-10T06:30:25,302 INFO [Listener at localhost/39735 {}] http.HttpServer(1219): Jetty bound to port 37431 2023-08-10T06:30:25,304 INFO [Listener at localhost/39735 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-08-10T06:30:25,335 INFO [Listener at localhost/39735 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:30:25,339 INFO [Listener at localhost/39735 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@1591e208{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/hadoop.log.dir/,AVAILABLE} 2023-08-10T06:30:25,339 INFO [Listener at localhost/39735 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:30:25,340 INFO [Listener at localhost/39735 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@496f5e72{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-08-10T06:30:25,413 INFO [Listener at localhost/39735 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-08-10T06:30:25,428 INFO [Listener at localhost/39735 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-08-10T06:30:25,428 INFO [Listener at localhost/39735 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-08-10T06:30:25,430 INFO [Listener at localhost/39735 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-08-10T06:30:25,437 INFO [Listener at localhost/39735 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:30:26,567 INFO [Listener at localhost/39735 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@233c5f70{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-08-10T06:30:26,579 INFO [Listener at localhost/39735 {}] server.AbstractConnector(333): Started ServerConnector@af7902f{HTTP/1.1, (http/1.1)}{0.0.0.0:37431} 2023-08-10T06:30:26,579 INFO [Listener at localhost/39735 {}] server.Server(415): Started @8268ms 2023-08-10T06:30:26,583 INFO [Listener at localhost/39735 {}] master.HMaster(484): hbase.rootdir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a, hbase.cluster.distributed=false 2023-08-10T06:30:26,643 INFO [Listener at localhost/39735 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-08-10T06:30:26,643 INFO [Listener at localhost/39735 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-08-10T06:30:26,643 INFO [Listener at localhost/39735 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-08-10T06:30:26,643 INFO [Listener at localhost/39735 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-08-10T06:30:26,643 INFO [Listener at localhost/39735 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-08-10T06:30:26,643 INFO [Listener at localhost/39735 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-08-10T06:30:26,651 INFO [Listener at localhost/39735 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-08-10T06:30:26,653 INFO [Listener at localhost/39735 {}] ipc.NettyRpcServer(177): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-08-10T06:30:26,654 INFO [Listener at localhost/39735 {}] ipc.NettyRpcServer(144): Bind to /172.31.12.81:40209 2023-08-10T06:30:26,657 INFO [Listener at localhost/39735 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-08-10T06:30:26,664 DEBUG [Listener at localhost/39735 {}] mob.MobFileCache(120): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-08-10T06:30:26,665 INFO [Listener at localhost/39735 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:30:26,668 INFO [Listener at localhost/39735 {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:30:26,670 INFO [Listener at localhost/39735 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:40209 connecting to ZooKeeper ensemble=127.0.0.1:55224 2023-08-10T06:30:26,674 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): regionserver:402090x0, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-08-10T06:30:26,675 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:40209-0x10060ecfb4e0001 connected 2023-08-10T06:30:26,675 DEBUG [Listener at localhost/39735 {}] zookeeper.ZKUtil(113): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-08-10T06:30:26,676 DEBUG [Listener at localhost/39735 {}] zookeeper.ZKUtil(113): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-08-10T06:30:26,677 DEBUG [Listener at localhost/39735 {}] zookeeper.ZKUtil(113): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-08-10T06:30:26,678 DEBUG [Listener at localhost/39735 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40209 2023-08-10T06:30:26,678 DEBUG [Listener at localhost/39735 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40209 2023-08-10T06:30:26,678 DEBUG [Listener at localhost/39735 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40209 2023-08-10T06:30:26,679 DEBUG [Listener at localhost/39735 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40209 2023-08-10T06:30:26,679 DEBUG [Listener at localhost/39735 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40209 2023-08-10T06:30:26,684 INFO [Listener at localhost/39735 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-08-10T06:30:26,684 INFO [Listener at localhost/39735 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-08-10T06:30:26,684 INFO [Listener at localhost/39735 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-08-10T06:30:26,685 INFO [Listener at localhost/39735 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-08-10T06:30:26,685 INFO [Listener at localhost/39735 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-08-10T06:30:26,685 INFO [Listener at localhost/39735 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-08-10T06:30:26,686 INFO [Listener at localhost/39735 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-08-10T06:30:26,688 INFO [Listener at localhost/39735 {}] http.HttpServer(1219): Jetty bound to port 42085 2023-08-10T06:30:26,688 INFO [Listener at localhost/39735 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-08-10T06:30:26,690 INFO [Listener at localhost/39735 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:30:26,690 INFO [Listener at localhost/39735 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5fac4aa4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/hadoop.log.dir/,AVAILABLE} 2023-08-10T06:30:26,690 INFO [Listener at localhost/39735 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:30:26,691 INFO [Listener at localhost/39735 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@3e6c1de8{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-08-10T06:30:26,702 INFO [Listener at localhost/39735 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-08-10T06:30:26,703 INFO [Listener at localhost/39735 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-08-10T06:30:26,703 INFO [Listener at localhost/39735 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-08-10T06:30:26,703 INFO [Listener at localhost/39735 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-08-10T06:30:26,704 INFO [Listener at localhost/39735 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-08-10T06:30:26,709 INFO [Listener at localhost/39735 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@31f20006{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-08-10T06:30:26,710 INFO [Listener at localhost/39735 {}] server.AbstractConnector(333): Started ServerConnector@461e973d{HTTP/1.1, (http/1.1)}{0.0.0.0:42085} 2023-08-10T06:30:26,710 INFO [Listener at localhost/39735 {}] server.Server(415): Started @8399ms 2023-08-10T06:30:26,715 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-08-10T06:30:26,718 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@528f8ba6{HTTP/1.1, (http/1.1)}{0.0.0.0:45229} 2023-08-10T06:30:26,718 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @8407ms 2023-08-10T06:30:26,718 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2387): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,46449,1691649023646 2023-08-10T06:30:26,730 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-08-10T06:30:26,730 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-08-10T06:30:26,732 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,46449,1691649023646 2023-08-10T06:30:26,743 DEBUG [M:0;jenkins-hbase3:46449 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:46449 2023-08-10T06:30:26,752 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-08-10T06:30:26,752 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-08-10T06:30:26,752 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:30:26,753 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:30:26,753 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Set watcher on existing znode=/1/master 2023-08-10T06:30:26,755 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,46449,1691649023646 from backup master directory 2023-08-10T06:30:26,755 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Set watcher on existing znode=/1/master 2023-08-10T06:30:26,759 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,46449,1691649023646 2023-08-10T06:30:26,759 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-08-10T06:30:26,759 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-08-10T06:30:26,760 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-08-10T06:30:26,760 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,46449,1691649023646 2023-08-10T06:30:26,763 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-08-10T06:30:26,765 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-08-10T06:30:26,849 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(610): Created cluster ID file at hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/hbase.id with ID: a5f52e32-ff46-4dea-b671-78478bb55385 2023-08-10T06:30:26,890 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(337): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-08-10T06:30:26,907 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:30:26,907 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:30:26,950 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:30:26,952 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-08-10T06:30:26,962 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2023-08-10T06:30:26,967 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.RegionGroupingProvider(103): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2023-08-10T06:30:26,979 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(270): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-08-10T06:30:26,980 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(276): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-08-10T06:30:26,981 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(285): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:283) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:310) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.init(RegionGroupingProvider.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createProvider(WALFactory.java:186) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:201) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-08-10T06:30:26,992 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.RegionGroupingProvider.init(RegionGroupingProvider.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.createProvider(WALFactory.java:186) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:201) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:950) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:573) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:570) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-08-10T06:30:27,093 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/data/master/store 2023-08-10T06:30:27,518 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-08-10T06:30:27,518 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:27,519 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-08-10T06:30:27,519 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:30:27,519 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:30:27,519 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-08-10T06:30:27,519 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:30:27,519 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:30:27,519 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-08-10T06:30:27,521 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/data/master/store/.initializing 2023-08-10T06:30:27,521 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/WALs/jenkins-hbase3.apache.org,46449,1691649023646 2023-08-10T06:30:27,524 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-08-10T06:30:27,533 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor jenkins-hbase3.apache.org,46449,1691649023646.regiongroup-0 2023-08-10T06:30:27,547 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C46449%2C1691649023646.jenkins-hbase3.apache.org%2C46449%2C1691649023646.regiongroup-0, suffix=, logDir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/WALs/jenkins-hbase3.apache.org,46449,1691649023646, archiveDir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/oldWALs, maxLogs=10 2023-08-10T06:30:27,575 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/WALs/jenkins-hbase3.apache.org,46449,1691649023646/jenkins-hbase3.apache.org%2C46449%2C1691649023646.jenkins-hbase3.apache.org%2C46449%2C1691649023646.regiongroup-0.1691649027553, exclude list is [], retry=0 2023-08-10T06:30:27,607 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK] 2023-08-10T06:30:27,615 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:346) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:430) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:424) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:120) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:482) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:477) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-08-10T06:30:27,693 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/WALs/jenkins-hbase3.apache.org,46449,1691649023646/jenkins-hbase3.apache.org%2C46449%2C1691649023646.jenkins-hbase3.apache.org%2C46449%2C1691649023646.regiongroup-0.1691649027553 2023-08-10T06:30:27,694 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK]] 2023-08-10T06:30:27,694 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:30:27,695 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:27,699 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:30:27,701 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:30:27,766 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:30:27,809 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-08-10T06:30:27,814 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:27,818 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:30:27,818 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:30:27,821 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-08-10T06:30:27,821 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:27,822 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:30:27,822 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:30:27,824 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-08-10T06:30:27,825 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:27,825 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:30:27,826 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:30:27,828 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-08-10T06:30:27,828 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:27,829 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:30:27,834 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:30:27,835 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:30:27,850 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-08-10T06:30:27,854 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-08-10T06:30:27,858 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:30:27,859 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10000846880, jitterRate=-0.06859855353832245}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-08-10T06:30:27,866 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-08-10T06:30:27,867 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-08-10T06:30:27,904 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30f4cb80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:27,962 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(859): No meta location available on zookeeper, skip migrating... 2023-08-10T06:30:27,984 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-08-10T06:30:27,984 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(564): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-08-10T06:30:27,987 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-08-10T06:30:27,988 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(584): Recovered RegionProcedureStore lease in 1 msec 2023-08-10T06:30:27,995 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(598): Loaded RegionProcedureStore in 6 msec 2023-08-10T06:30:27,995 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-08-10T06:30:28,030 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-08-10T06:30:28,044 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-08-10T06:30:28,047 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-08-10T06:30:28,050 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(150): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-08-10T06:30:28,051 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-08-10T06:30:28,052 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-08-10T06:30:28,055 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-08-10T06:30:28,061 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-08-10T06:30:28,064 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-08-10T06:30:28,065 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-08-10T06:30:28,066 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-08-10T06:30:28,081 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-08-10T06:30:28,082 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-08-10T06:30:28,087 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-08-10T06:30:28,087 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-08-10T06:30:28,087 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:30:28,087 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:30:28,088 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(803): Active/primary master=jenkins-hbase3.apache.org,46449,1691649023646, sessionid=0x10060ecfb4e0000, setting cluster-up flag (Was=false) 2023-08-10T06:30:28,098 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-08-10T06:30:28,106 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:30:28,106 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:30:28,114 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-08-10T06:30:28,116 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,46449,1691649023646 2023-08-10T06:30:28,119 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:30:28,119 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:30:28,125 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-08-10T06:30:28,126 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,46449,1691649023646 2023-08-10T06:30:28,130 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:30:28,130 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:30:28,138 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-08-10T06:30:28,139 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,46449,1691649023646 2023-08-10T06:30:28,223 DEBUG [RS:0;jenkins-hbase3:40209 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:40209 2023-08-10T06:30:28,225 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(997): ClusterId : a5f52e32-ff46-4dea-b671-78478bb55385 2023-08-10T06:30:28,227 INFO [RS:0;jenkins-hbase3:40209 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-08-10T06:30:28,229 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-08-10T06:30:28,235 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-08-10T06:30:28,235 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-08-10T06:30:28,239 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-08-10T06:30:28,239 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-08-10T06:30:28,242 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-08-10T06:30:28,243 DEBUG [RS:0;jenkins-hbase3:40209 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b8e1764, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:28,245 DEBUG [RS:0;jenkins-hbase3:40209 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2382e507, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-08-10T06:30:28,250 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-08-10T06:30:28,250 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-08-10T06:30:28,250 DEBUG [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-08-10T06:30:28,254 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,46449,1691649023646 with isa=jenkins-hbase3.apache.org/172.31.12.81:40209, startcode=1691649026642 2023-08-10T06:30:28,272 DEBUG [RS:0;jenkins-hbase3:40209 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-08-10T06:30:28,281 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-08-10T06:30:28,290 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-08-10T06:30:28,295 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-08-10T06:30:28,305 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,46449,1691649023646 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-08-10T06:30:28,310 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-08-10T06:30:28,310 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-08-10T06:30:28,310 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-08-10T06:30:28,310 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-08-10T06:30:28,310 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-08-10T06:30:28,310 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:30:28,311 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-08-10T06:30:28,311 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:30:28,312 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1691649058312 2023-08-10T06:30:28,315 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-08-10T06:30:28,321 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-08-10T06:30:28,321 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-08-10T06:30:28,322 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-08-10T06:30:28,327 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:28,328 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-08-10T06:30:28,336 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-08-10T06:30:28,336 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-08-10T06:30:28,337 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-08-10T06:30:28,338 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-08-10T06:30:28,338 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-08-10T06:30:28,343 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-08-10T06:30:28,346 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-08-10T06:30:28,349 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-08-10T06:30:28,350 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-08-10T06:30:28,357 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40531, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-08-10T06:30:28,358 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x2b16221b to 127.0.0.1:55224 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:30:28,359 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-08-10T06:30:28,359 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a 2023-08-10T06:30:28,377 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46449 {}] ipc.MetricsHBaseServer(144): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3180) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:588) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-08-10T06:30:28,394 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b86335f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:28,394 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-08-10T06:30:28,397 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:28,399 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-08-10T06:30:28,399 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-08-10T06:30:28,406 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1691649028402,5,FailOnTimeoutGroup] 2023-08-10T06:30:28,406 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-08-10T06:30:28,406 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1691649028406,5,FailOnTimeoutGroup] 2023-08-10T06:30:28,406 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-08-10T06:30:28,407 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1634): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-08-10T06:30:28,408 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-08-10T06:30:28,409 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-08-10T06:30:28,409 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-08-10T06:30:28,409 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:28,410 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:30:28,410 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-08-10T06:30:28,412 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-08-10T06:30:28,413 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:28,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:30:28,414 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-08-10T06:30:28,416 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-08-10T06:30:28,416 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:28,417 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:30:28,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/meta/1588230740 2023-08-10T06:30:28,420 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/meta/1588230740 2023-08-10T06:30:28,422 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-08-10T06:30:28,425 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-08-10T06:30:28,430 DEBUG [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(3042): Master is not running yet 2023-08-10T06:30:28,430 WARN [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-08-10T06:30:28,430 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:30:28,431 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11582124320, jitterRate=0.07866938412189484}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-08-10T06:30:28,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-08-10T06:30:28,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-08-10T06:30:28,433 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-08-10T06:30:28,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-08-10T06:30:28,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-08-10T06:30:28,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-08-10T06:30:28,435 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-08-10T06:30:28,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-08-10T06:30:28,438 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-08-10T06:30:28,439 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-08-10T06:30:28,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-08-10T06:30:28,461 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-08-10T06:30:28,465 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-08-10T06:30:28,531 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(3023): reportForDuty to master=jenkins-hbase3.apache.org,46449,1691649023646 with isa=jenkins-hbase3.apache.org/172.31.12.81:40209, startcode=1691649026642 2023-08-10T06:30:28,536 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46449 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:28,549 DEBUG [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a 2023-08-10T06:30:28,549 DEBUG [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:43579 2023-08-10T06:30:28,549 DEBUG [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=37431 2023-08-10T06:30:28,555 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-08-10T06:30:28,555 DEBUG [RS:0;jenkins-hbase3:40209 {}] zookeeper.ZKUtil(111): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:28,555 WARN [RS:0;jenkins-hbase3:40209 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-08-10T06:30:28,555 INFO [RS:0;jenkins-hbase3:40209 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2023-08-10T06:30:28,556 INFO [RS:0;jenkins-hbase3:40209 {}] wal.RegionGroupingProvider(103): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2023-08-10T06:30:28,556 DEBUG [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:28,559 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,40209,1691649026642] 2023-08-10T06:30:28,573 DEBUG [RS:0;jenkins-hbase3:40209 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-08-10T06:30:28,582 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-08-10T06:30:28,600 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-08-10T06:30:28,606 INFO [RS:0;jenkins-hbase3:40209 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-08-10T06:30:28,606 INFO [RS:0;jenkins-hbase3:40209 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-08-10T06:30:28,607 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-08-10T06:30:28,616 WARN [jenkins-hbase3:46449 {}] assignment.AssignmentManager(2282): No servers available; cannot place 1 unassigned regions. 2023-08-10T06:30:28,617 INFO [RS:0;jenkins-hbase3:40209 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-08-10T06:30:28,617 DEBUG [RS:0;jenkins-hbase3:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:30:28,618 DEBUG [RS:0;jenkins-hbase3:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:30:28,618 DEBUG [RS:0;jenkins-hbase3:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:30:28,618 DEBUG [RS:0;jenkins-hbase3:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:30:28,618 DEBUG [RS:0;jenkins-hbase3:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:30:28,618 DEBUG [RS:0;jenkins-hbase3:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-08-10T06:30:28,618 DEBUG [RS:0;jenkins-hbase3:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:30:28,618 DEBUG [RS:0;jenkins-hbase3:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:30:28,618 DEBUG [RS:0;jenkins-hbase3:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:30:28,619 DEBUG [RS:0;jenkins-hbase3:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:30:28,619 DEBUG [RS:0;jenkins-hbase3:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-08-10T06:30:28,619 DEBUG [RS:0;jenkins-hbase3:40209 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-08-10T06:30:28,620 INFO [RS:0;jenkins-hbase3:40209 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-08-10T06:30:28,620 INFO [RS:0;jenkins-hbase3:40209 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-08-10T06:30:28,620 INFO [RS:0;jenkins-hbase3:40209 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-08-10T06:30:28,620 INFO [RS:0;jenkins-hbase3:40209 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-08-10T06:30:28,620 INFO [RS:0;jenkins-hbase3:40209 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,40209,1691649026642-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-08-10T06:30:28,636 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-08-10T06:30:28,639 INFO [RS:0;jenkins-hbase3:40209 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,40209,1691649026642-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-08-10T06:30:28,653 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,40209,1691649026642 started 2023-08-10T06:30:28,653 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,40209,1691649026642, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:40209, sessionid=0x10060ecfb4e0001 2023-08-10T06:30:28,654 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-08-10T06:30:28,654 DEBUG [RS:0;jenkins-hbase3:40209 {}] flush.RegionServerFlushTableProcedureManager(106): Start region server flush procedure manager jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:28,654 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,40209,1691649026642' 2023-08-10T06:30:28,654 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-08-10T06:30:28,655 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-08-10T06:30:28,655 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-08-10T06:30:28,655 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-08-10T06:30:28,655 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,40209,1691649026642' 2023-08-10T06:30:28,656 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:30:28,656 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:30:28,657 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-08-10T06:30:28,657 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-08-10T06:30:28,657 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-08-10T06:30:28,657 DEBUG [RS:0;jenkins-hbase3:40209 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:28,657 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,40209,1691649026642' 2023-08-10T06:30:28,657 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-08-10T06:30:28,657 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-08-10T06:30:28,658 DEBUG [RS:0;jenkins-hbase3:40209 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-08-10T06:30:28,658 INFO [RS:0;jenkins-hbase3:40209 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-08-10T06:30:28,658 INFO [RS:0;jenkins-hbase3:40209 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-08-10T06:30:28,768 INFO [RS:0;jenkins-hbase3:40209 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-08-10T06:30:28,769 INFO [RS:0;jenkins-hbase3:40209 {}] monitor.StreamSlowMonitor(122): New stream slow monitor jenkins-hbase3.apache.org,40209,1691649026642.regiongroup-0 2023-08-10T06:30:28,772 INFO [RS:0;jenkins-hbase3:40209 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0, suffix=, logDir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642, archiveDir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/oldWALs, maxLogs=32 2023-08-10T06:30:28,784 DEBUG [RS:0;jenkins-hbase3:40209 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649028773, exclude list is [], retry=0 2023-08-10T06:30:28,789 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK] 2023-08-10T06:30:28,793 INFO [RS:0;jenkins-hbase3:40209 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649028773 2023-08-10T06:30:28,794 DEBUG [RS:0;jenkins-hbase3:40209 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK]] 2023-08-10T06:30:28,867 DEBUG [jenkins-hbase3:46449 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-08-10T06:30:28,880 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:28,885 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,40209,1691649026642, state=OPENING 2023-08-10T06:30:28,896 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-08-10T06:30:28,900 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:30:28,900 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:30:28,901 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-08-10T06:30:28,901 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-08-10T06:30:28,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,40209,1691649026642}] 2023-08-10T06:30:29,096 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:29,099 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-08-10T06:30:29,103 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48340, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-08-10T06:30:29,116 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-08-10T06:30:29,117 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.RegionGroupingProvider 2023-08-10T06:30:29,117 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.RegionGroupingProvider(103): Instantiating RegionGroupingStrategy of type class org.apache.hadoop.hbase.wal.BoundedGroupingStrategy 2023-08-10T06:30:29,118 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-08-10T06:30:29,118 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-08-10T06:30:29,121 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta, suffix=.meta, logDir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642, archiveDir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/oldWALs, maxLogs=32 2023-08-10T06:30:29,134 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649029122.meta, exclude list is [], retry=0 2023-08-10T06:30:29,138 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK] 2023-08-10T06:30:29,141 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649029122.meta 2023-08-10T06:30:29,141 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK]] 2023-08-10T06:30:29,141 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:30:29,146 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:30:29,147 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-08-10T06:30:29,163 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-08-10T06:30:29,163 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-08-10T06:30:29,168 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-08-10T06:30:29,168 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:29,168 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-08-10T06:30:29,168 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-08-10T06:30:29,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-08-10T06:30:29,173 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-08-10T06:30:29,173 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:29,173 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:30:29,174 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-08-10T06:30:29,175 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-08-10T06:30:29,175 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:29,175 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:30:29,176 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-08-10T06:30:29,177 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-08-10T06:30:29,177 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:29,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-08-10T06:30:29,179 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/meta/1588230740 2023-08-10T06:30:29,181 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/meta/1588230740 2023-08-10T06:30:29,184 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-08-10T06:30:29,186 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-08-10T06:30:29,187 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9693508000, jitterRate=-0.09722171723842621}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-08-10T06:30:29,188 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-08-10T06:30:29,200 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1691649029088 2023-08-10T06:30:29,218 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:meta,,1.1588230740 2023-08-10T06:30:29,219 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-08-10T06:30:29,220 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:29,222 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,40209,1691649026642, state=OPEN 2023-08-10T06:30:29,224 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-08-10T06:30:29,224 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-08-10T06:30:29,224 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-08-10T06:30:29,224 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-08-10T06:30:29,228 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=3, resume processing ppid=2 2023-08-10T06:30:29,228 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,40209,1691649026642 in 320 msec 2023-08-10T06:30:29,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=2, resume processing ppid=1 2023-08-10T06:30:29,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 782 msec 2023-08-10T06:30:29,380 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.0410 sec 2023-08-10T06:30:29,380 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1062): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1691649029380, completionTime=-1 2023-08-10T06:30:29,381 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-08-10T06:30:29,381 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-08-10T06:30:29,447 DEBUG [hconnection-0x3e2a44a9-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:29,451 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:48342, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:29,467 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-08-10T06:30:29,467 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1691649089467 2023-08-10T06:30:29,467 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1691649149467 2023-08-10T06:30:29,468 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 86 msec 2023-08-10T06:30:29,493 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46449,1691649023646-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-08-10T06:30:29,493 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46449,1691649023646-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-08-10T06:30:29,493 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46449,1691649023646-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-08-10T06:30:29,495 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:46449, period=300000, unit=MILLISECONDS is enabled. 2023-08-10T06:30:29,496 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-08-10T06:30:29,503 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-08-10T06:30:29,518 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-08-10T06:30:29,519 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2367): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-08-10T06:30:29,529 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-08-10T06:30:29,532 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:30:29,534 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:29,536 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:30:29,956 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => faa733e2577a03b5cd26e4424a6f4087, NAME => 'hbase:namespace,,1691649029519.faa733e2577a03b5cd26e4424a6f4087.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a 2023-08-10T06:30:30,367 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1691649029519.faa733e2577a03b5cd26e4424a6f4087.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:30,367 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing faa733e2577a03b5cd26e4424a6f4087, disabling compactions & flushes 2023-08-10T06:30:30,367 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1691649029519.faa733e2577a03b5cd26e4424a6f4087. 2023-08-10T06:30:30,368 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1691649029519.faa733e2577a03b5cd26e4424a6f4087. 2023-08-10T06:30:30,368 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1691649029519.faa733e2577a03b5cd26e4424a6f4087. after waiting 0 ms 2023-08-10T06:30:30,368 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1691649029519.faa733e2577a03b5cd26e4424a6f4087. 2023-08-10T06:30:30,368 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1691649029519.faa733e2577a03b5cd26e4424a6f4087. 2023-08-10T06:30:30,368 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for faa733e2577a03b5cd26e4424a6f4087: 2023-08-10T06:30:30,370 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:30:30,384 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1691649029519.faa733e2577a03b5cd26e4424a6f4087.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1691649030372"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649030372"}]},"ts":"1691649030372"} 2023-08-10T06:30:30,432 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:30:30,435 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:30:30,441 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649030435"}]},"ts":"1691649030435"} 2023-08-10T06:30:30,445 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-08-10T06:30:30,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=faa733e2577a03b5cd26e4424a6f4087, ASSIGN}] 2023-08-10T06:30:30,453 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=faa733e2577a03b5cd26e4424a6f4087, ASSIGN 2023-08-10T06:30:30,454 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=faa733e2577a03b5cd26e4424a6f4087, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40209,1691649026642; forceNewPlan=false, retain=false 2023-08-10T06:30:30,605 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=faa733e2577a03b5cd26e4424a6f4087, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:30,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure faa733e2577a03b5cd26e4424a6f4087, server=jenkins-hbase3.apache.org,40209,1691649026642}] 2023-08-10T06:30:30,770 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1691649029519.faa733e2577a03b5cd26e4424a6f4087. 2023-08-10T06:30:30,770 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-08-10T06:30:30,771 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] monitor.StreamSlowMonitor(122): New stream slow monitor jenkins-hbase3.apache.org,40209,1691649026642.regiongroup-1 2023-08-10T06:30:30,773 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1, suffix=, logDir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642, archiveDir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/oldWALs, maxLogs=32 2023-08-10T06:30:30,785 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649030774, exclude list is [], retry=0 2023-08-10T06:30:30,789 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK] 2023-08-10T06:30:30,793 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649030774 2023-08-10T06:30:30,793 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK]] 2023-08-10T06:30:30,793 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => faa733e2577a03b5cd26e4424a6f4087, NAME => 'hbase:namespace,,1691649029519.faa733e2577a03b5cd26e4424a6f4087.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:30:30,794 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:30:30,794 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace faa733e2577a03b5cd26e4424a6f4087 2023-08-10T06:30:30,794 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1691649029519.faa733e2577a03b5cd26e4424a6f4087.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:30,795 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for faa733e2577a03b5cd26e4424a6f4087 2023-08-10T06:30:30,795 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for faa733e2577a03b5cd26e4424a6f4087 2023-08-10T06:30:30,796 INFO [StoreOpener-faa733e2577a03b5cd26e4424a6f4087-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region faa733e2577a03b5cd26e4424a6f4087 2023-08-10T06:30:30,798 INFO [StoreOpener-faa733e2577a03b5cd26e4424a6f4087-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region faa733e2577a03b5cd26e4424a6f4087 columnFamilyName info 2023-08-10T06:30:30,798 DEBUG [StoreOpener-faa733e2577a03b5cd26e4424a6f4087-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:30,799 INFO [StoreOpener-faa733e2577a03b5cd26e4424a6f4087-1 {}] regionserver.HStore(324): Store=faa733e2577a03b5cd26e4424a6f4087/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:30:30,800 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/namespace/faa733e2577a03b5cd26e4424a6f4087 2023-08-10T06:30:30,801 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/namespace/faa733e2577a03b5cd26e4424a6f4087 2023-08-10T06:30:30,805 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for faa733e2577a03b5cd26e4424a6f4087 2023-08-10T06:30:30,808 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/namespace/faa733e2577a03b5cd26e4424a6f4087/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:30:30,809 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened faa733e2577a03b5cd26e4424a6f4087; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11448570720, jitterRate=0.06623123586177826}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:30:30,810 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for faa733e2577a03b5cd26e4424a6f4087: 2023-08-10T06:30:30,812 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2552): Post open deploy tasks for hbase:namespace,,1691649029519.faa733e2577a03b5cd26e4424a6f4087., pid=6, masterSystemTime=1691649030764 2023-08-10T06:30:30,815 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2579): Finished post open deploy task for hbase:namespace,,1691649029519.faa733e2577a03b5cd26e4424a6f4087. 2023-08-10T06:30:30,815 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1691649029519.faa733e2577a03b5cd26e4424a6f4087. 2023-08-10T06:30:30,816 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=faa733e2577a03b5cd26e4424a6f4087, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:30,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=6, resume processing ppid=5 2023-08-10T06:30:30,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure faa733e2577a03b5cd26e4424a6f4087, server=jenkins-hbase3.apache.org,40209,1691649026642 in 209 msec 2023-08-10T06:30:30,826 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=5, resume processing ppid=4 2023-08-10T06:30:30,826 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=faa733e2577a03b5cd26e4424a6f4087, ASSIGN in 373 msec 2023-08-10T06:30:30,827 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:30:30,827 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649030827"}]},"ts":"1691649030827"} 2023-08-10T06:30:30,830 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-08-10T06:30:30,833 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:30:30,834 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-08-10T06:30:30,835 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:30:30,835 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-08-10T06:30:30,835 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-08-10T06:30:30,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.3130 sec 2023-08-10T06:30:30,891 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-08-10T06:30:30,907 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:30:30,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 34 msec 2023-08-10T06:30:30,916 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1033): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-08-10T06:30:30,929 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:30:30,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 16 msec 2023-08-10T06:30:30,943 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-08-10T06:30:30,945 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-08-10T06:30:30,945 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1192): Master has completed initialization 4.185sec 2023-08-10T06:30:30,947 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-08-10T06:30:30,949 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-08-10T06:30:30,950 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-08-10T06:30:30,951 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-08-10T06:30:30,951 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-08-10T06:30:30,953 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46449,1691649023646-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-08-10T06:30:30,953 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,46449,1691649023646-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-08-10T06:30:30,966 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1295): Balancer post startup initialization complete, took 0 seconds 2023-08-10T06:30:30,968 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-08-10T06:30:30,977 DEBUG [Listener at localhost/39735 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x689ffcba to 127.0.0.1:55224 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:30:30,985 DEBUG [Listener at localhost/39735 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f9f9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:30,990 DEBUG [Listener at localhost/39735 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-08-10T06:30:30,990 DEBUG [Listener at localhost/39735 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-08-10T06:30:31,003 DEBUG [hconnection-0x1df1c5f3-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:31,014 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:35778, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:31,024 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,46449,1691649023646 2023-08-10T06:30:31,024 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-08-10T06:30:31,024 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/test.cache.data in system properties and HBase conf 2023-08-10T06:30:31,024 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/hadoop.tmp.dir in system properties and HBase conf 2023-08-10T06:30:31,025 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/hadoop.log.dir in system properties and HBase conf 2023-08-10T06:30:31,025 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/mapreduce.cluster.local.dir in system properties and HBase conf 2023-08-10T06:30:31,025 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-08-10T06:30:31,025 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-08-10T06:30:31,025 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-08-10T06:30:31,025 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-08-10T06:30:31,025 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-08-10T06:30:31,025 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-08-10T06:30:31,025 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-08-10T06:30:31,025 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-08-10T06:30:31,025 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-08-10T06:30:31,025 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/dfs.journalnode.edits.dir in system properties and HBase conf 2023-08-10T06:30:31,026 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-08-10T06:30:31,026 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/nfs.dump.dir in system properties and HBase conf 2023-08-10T06:30:31,026 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/java.io.tmpdir in system properties and HBase conf 2023-08-10T06:30:31,026 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/dfs.journalnode.edits.dir in system properties and HBase conf 2023-08-10T06:30:31,026 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-08-10T06:30:31,026 INFO [Listener at localhost/39735 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-08-10T06:30:32,507 WARN [Thread-251 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:30:32,863 INFO [Thread-251 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:30:32,869 INFO [Thread-251 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_36519_jobhistory____.pmdg7w/webapp 2023-08-10T06:30:33,031 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-08-10T06:30:33,031 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-08-10T06:30:33,031 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-08-10T06:30:33,034 INFO [Thread-251 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-08-10T06:30:33,118 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:30:33,234 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:30:33,488 INFO [Thread-251 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-08-10T06:30:33,517 INFO [Thread-251 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:36519 2023-08-10T06:30:33,812 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-08-10T06:30:33,813 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-08-10T06:30:33,820 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-08-10T06:30:33,820 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-08-10T06:30:33,821 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:30:33,821 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-08-10T06:30:33,822 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-08-10T06:30:33,822 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-08-10T06:30:33,822 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-08-10T06:30:33,822 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-08-10T06:30:34,005 WARN [Listener at jenkins-hbase3.apache.org/38575 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:30:34,093 WARN [Listener at jenkins-hbase3.apache.org/38575 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-08-10T06:30:34,093 INFO [Listener at jenkins-hbase3.apache.org/38575 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:30:34,099 INFO [Listener at jenkins-hbase3.apache.org/38575 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_34667_cluster____.s7xub/webapp 2023-08-10T06:30:34,168 INFO [Listener at jenkins-hbase3.apache.org/38575 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-08-10T06:30:34,169 INFO [Listener at jenkins-hbase3.apache.org/38575 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-08-10T06:30:34,169 INFO [Listener at jenkins-hbase3.apache.org/38575 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-08-10T06:30:34,169 INFO [Listener at jenkins-hbase3.apache.org/38575 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-08-10T06:30:34,185 INFO [Listener at jenkins-hbase3.apache.org/38575 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:30:34,392 INFO [Listener at jenkins-hbase3.apache.org/38575 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:30:34,486 INFO [Listener at jenkins-hbase3.apache.org/38575 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:30:34,493 INFO [Listener at jenkins-hbase3.apache.org/38575 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:34667 2023-08-10T06:30:34,703 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:30:34,772 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-08-10T06:30:34,773 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-08-10T06:30:34,871 WARN [Listener at jenkins-hbase3.apache.org/33523 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:30:34,894 WARN [Listener at jenkins-hbase3.apache.org/33523 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-08-10T06:30:34,895 INFO [Listener at jenkins-hbase3.apache.org/33523 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:30:34,904 INFO [Listener at jenkins-hbase3.apache.org/33523 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_33983_node____tlr8hf/webapp 2023-08-10T06:30:34,956 INFO [Listener at jenkins-hbase3.apache.org/33523 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-08-10T06:30:34,956 INFO [Listener at jenkins-hbase3.apache.org/33523 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-08-10T06:30:34,956 INFO [Listener at jenkins-hbase3.apache.org/33523 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-08-10T06:30:34,956 INFO [Listener at jenkins-hbase3.apache.org/33523 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-08-10T06:30:34,963 INFO [Listener at jenkins-hbase3.apache.org/33523 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:30:34,985 INFO [Listener at jenkins-hbase3.apache.org/33523 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:30:35,047 INFO [Listener at jenkins-hbase3.apache.org/33523 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:30:35,056 INFO [Listener at jenkins-hbase3.apache.org/33523 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:33983 2023-08-10T06:30:35,191 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-08-10T06:30:35,200 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-08-10T06:30:35,201 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] log.Slf4jLog(67): jetty-6.1.26 2023-08-10T06:30:35,210 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_42291_node____32033f/webapp 2023-08-10T06:30:35,276 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-08-10T06:30:35,276 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-08-10T06:30:35,276 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-08-10T06:30:35,276 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-08-10T06:30:35,284 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:30:35,299 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:30:35,357 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-08-10T06:30:35,366 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:42291 2023-08-10T06:30:35,399 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-08-10T06:30:35,399 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:43579/backupUT 2023-08-10T06:30:35,411 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:30:35,415 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37400, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:30:35,425 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-08-10T06:30:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-08-10T06:30:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-08-10T06:30:35,450 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:30:35,454 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 27 msec 2023-08-10T06:30:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=9 2023-08-10T06:30:35,547 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-08-10T06:30:35,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-08-10T06:30:35,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-08-10T06:30:35,560 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:30:35,567 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 18 msec 2023-08-10T06:30:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=10 2023-08-10T06:30:35,657 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-08-10T06:30:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-08-10T06:30:35,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-08-10T06:30:35,671 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:30:35,676 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 16 msec 2023-08-10T06:30:35,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=11 2023-08-10T06:30:35,768 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-08-10T06:30:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-08-10T06:30:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-08-10T06:30:35,781 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:30:35,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 16 msec 2023-08-10T06:30:35,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=12 2023-08-10T06:30:35,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'test-1691649035399', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:30:35,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1691649035399 2023-08-10T06:30:35,893 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1691649035399 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:30:35,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1691649035399" procId is: 13 2023-08-10T06:30:35,894 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:35,895 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1691649035399 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:30:35,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:30:36,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:30:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:30:36,331 INFO [RegionOpenAndInit-test-1691649035399-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 41ac6ed181c2526bbcdb3ef2c04e3054, NAME => 'test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1691649035399', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a 2023-08-10T06:30:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:30:36,749 DEBUG [RegionOpenAndInit-test-1691649035399-pool-0 {}] regionserver.HRegion(888): Instantiated test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:36,750 DEBUG [RegionOpenAndInit-test-1691649035399-pool-0 {}] regionserver.HRegion(1668): Closing 41ac6ed181c2526bbcdb3ef2c04e3054, disabling compactions & flushes 2023-08-10T06:30:36,750 INFO [RegionOpenAndInit-test-1691649035399-pool-0 {}] regionserver.HRegion(1690): Closing region test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:30:36,750 DEBUG [RegionOpenAndInit-test-1691649035399-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:30:36,750 DEBUG [RegionOpenAndInit-test-1691649035399-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. after waiting 0 ms 2023-08-10T06:30:36,750 DEBUG [RegionOpenAndInit-test-1691649035399-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:30:36,750 INFO [RegionOpenAndInit-test-1691649035399-pool-0 {}] regionserver.HRegion(1909): Closed test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:30:36,750 DEBUG [RegionOpenAndInit-test-1691649035399-pool-0 {}] regionserver.HRegion(1622): Region close journal for 41ac6ed181c2526bbcdb3ef2c04e3054: 2023-08-10T06:30:36,751 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1691649035399 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:30:36,752 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1691649036752"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649036752"}]},"ts":"1691649036752"} 2023-08-10T06:30:36,754 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:30:36,755 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1691649035399 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:30:36,756 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1691649035399","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649036755"}]},"ts":"1691649036755"} 2023-08-10T06:30:36,757 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1691649035399, state=ENABLING in hbase:meta 2023-08-10T06:30:36,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1691649035399, region=41ac6ed181c2526bbcdb3ef2c04e3054, ASSIGN}] 2023-08-10T06:30:36,765 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1691649035399, region=41ac6ed181c2526bbcdb3ef2c04e3054, ASSIGN 2023-08-10T06:30:36,766 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1691649035399, region=41ac6ed181c2526bbcdb3ef2c04e3054, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40209,1691649026642; forceNewPlan=false, retain=false 2023-08-10T06:30:36,918 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=41ac6ed181c2526bbcdb3ef2c04e3054, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:36,921 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 41ac6ed181c2526bbcdb3ef2c04e3054, server=jenkins-hbase3.apache.org,40209,1691649026642}] 2023-08-10T06:30:37,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:30:37,081 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:30:37,082 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 41ac6ed181c2526bbcdb3ef2c04e3054, NAME => 'test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:30:37,082 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:30:37,082 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1691649035399 41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:30:37,082 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:37,082 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:30:37,082 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:30:37,085 INFO [StoreOpener-41ac6ed181c2526bbcdb3ef2c04e3054-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:30:37,088 INFO [StoreOpener-41ac6ed181c2526bbcdb3ef2c04e3054-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41ac6ed181c2526bbcdb3ef2c04e3054 columnFamilyName f 2023-08-10T06:30:37,088 DEBUG [StoreOpener-41ac6ed181c2526bbcdb3ef2c04e3054-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:37,089 INFO [StoreOpener-41ac6ed181c2526bbcdb3ef2c04e3054-1 {}] regionserver.HStore(324): Store=41ac6ed181c2526bbcdb3ef2c04e3054/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:30:37,091 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:30:37,091 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:30:37,096 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:30:37,101 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:30:37,102 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 41ac6ed181c2526bbcdb3ef2c04e3054; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10143384800, jitterRate=-0.05532367527484894}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:30:37,103 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 41ac6ed181c2526bbcdb3ef2c04e3054: 2023-08-10T06:30:37,105 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2552): Post open deploy tasks for test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054., pid=15, masterSystemTime=1691649037074 2023-08-10T06:30:37,108 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2579): Finished post open deploy task for test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:30:37,108 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:30:37,109 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=41ac6ed181c2526bbcdb3ef2c04e3054, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:37,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=15, resume processing ppid=14 2023-08-10T06:30:37,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 41ac6ed181c2526bbcdb3ef2c04e3054, server=jenkins-hbase3.apache.org,40209,1691649026642 in 191 msec 2023-08-10T06:30:37,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=14, resume processing ppid=13 2023-08-10T06:30:37,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1691649035399, region=41ac6ed181c2526bbcdb3ef2c04e3054, ASSIGN in 352 msec 2023-08-10T06:30:37,119 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1691649035399 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:30:37,119 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1691649035399","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649037119"}]},"ts":"1691649037119"} 2023-08-10T06:30:37,121 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1691649035399, state=ENABLED in hbase:meta 2023-08-10T06:30:37,124 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1691649035399 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:30:37,126 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1691649035399 in 1.2350 sec 2023-08-10T06:30:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=13 2023-08-10T06:30:38,009 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: default:test-1691649035399, procId: 13 completed 2023-08-10T06:30:38,010 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x36539b84 to 127.0.0.1:55224 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:30:38,016 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@370cb09b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:38,018 DEBUG [hconnection-0x2ddf115c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:38,021 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:35790, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:38,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40209 {}] regionserver.HRegion(8210): writing data to region test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. with WAL disabled. Data may be lost in the event of a crash. 2023-08-10T06:30:38,127 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns2:test-16916490353991', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:30:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16916490353991 2023-08-10T06:30:38,131 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16916490353991 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:30:38,131 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:38,131 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16916490353991" procId is: 16 2023-08-10T06:30:38,132 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16916490353991 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:30:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:30:38,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:30:38,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:30:38,550 INFO [RegionOpenAndInit-ns2:test-16916490353991-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => ab0a54958c666e251775ab33a9efc343, NAME => 'ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16916490353991', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a 2023-08-10T06:30:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:30:38,968 DEBUG [RegionOpenAndInit-ns2:test-16916490353991-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:38,968 DEBUG [RegionOpenAndInit-ns2:test-16916490353991-pool-0 {}] regionserver.HRegion(1668): Closing ab0a54958c666e251775ab33a9efc343, disabling compactions & flushes 2023-08-10T06:30:38,968 INFO [RegionOpenAndInit-ns2:test-16916490353991-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343. 2023-08-10T06:30:38,968 DEBUG [RegionOpenAndInit-ns2:test-16916490353991-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343. 2023-08-10T06:30:38,968 DEBUG [RegionOpenAndInit-ns2:test-16916490353991-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343. after waiting 0 ms 2023-08-10T06:30:38,968 DEBUG [RegionOpenAndInit-ns2:test-16916490353991-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343. 2023-08-10T06:30:38,968 INFO [RegionOpenAndInit-ns2:test-16916490353991-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343. 2023-08-10T06:30:38,968 DEBUG [RegionOpenAndInit-ns2:test-16916490353991-pool-0 {}] regionserver.HRegion(1622): Region close journal for ab0a54958c666e251775ab33a9efc343: 2023-08-10T06:30:38,970 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16916490353991 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:30:38,970 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1691649038970"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649038970"}]},"ts":"1691649038970"} 2023-08-10T06:30:38,973 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:30:38,974 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16916490353991 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:30:38,974 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16916490353991","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649038974"}]},"ts":"1691649038974"} 2023-08-10T06:30:38,976 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16916490353991, state=ENABLING in hbase:meta 2023-08-10T06:30:38,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16916490353991, region=ab0a54958c666e251775ab33a9efc343, ASSIGN}] 2023-08-10T06:30:38,984 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16916490353991, region=ab0a54958c666e251775ab33a9efc343, ASSIGN 2023-08-10T06:30:38,985 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16916490353991, region=ab0a54958c666e251775ab33a9efc343, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40209,1691649026642; forceNewPlan=false, retain=false 2023-08-10T06:30:39,137 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=ab0a54958c666e251775ab33a9efc343, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:39,139 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure ab0a54958c666e251775ab33a9efc343, server=jenkins-hbase3.apache.org,40209,1691649026642}] 2023-08-10T06:30:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:30:39,298 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343. 2023-08-10T06:30:39,298 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => ab0a54958c666e251775ab33a9efc343, NAME => 'ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:30:39,299 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:30:39,299 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16916490353991 ab0a54958c666e251775ab33a9efc343 2023-08-10T06:30:39,299 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:39,299 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for ab0a54958c666e251775ab33a9efc343 2023-08-10T06:30:39,299 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for ab0a54958c666e251775ab33a9efc343 2023-08-10T06:30:39,301 INFO [StoreOpener-ab0a54958c666e251775ab33a9efc343-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region ab0a54958c666e251775ab33a9efc343 2023-08-10T06:30:39,303 INFO [StoreOpener-ab0a54958c666e251775ab33a9efc343-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab0a54958c666e251775ab33a9efc343 columnFamilyName f 2023-08-10T06:30:39,303 DEBUG [StoreOpener-ab0a54958c666e251775ab33a9efc343-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:39,304 INFO [StoreOpener-ab0a54958c666e251775ab33a9efc343-1 {}] regionserver.HStore(324): Store=ab0a54958c666e251775ab33a9efc343/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:30:39,305 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns2/test-16916490353991/ab0a54958c666e251775ab33a9efc343 2023-08-10T06:30:39,306 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns2/test-16916490353991/ab0a54958c666e251775ab33a9efc343 2023-08-10T06:30:39,310 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for ab0a54958c666e251775ab33a9efc343 2023-08-10T06:30:39,317 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns2/test-16916490353991/ab0a54958c666e251775ab33a9efc343/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:30:39,318 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened ab0a54958c666e251775ab33a9efc343; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10786750240, jitterRate=0.004594400525093079}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:30:39,318 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for ab0a54958c666e251775ab33a9efc343: 2023-08-10T06:30:39,319 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2552): Post open deploy tasks for ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343., pid=18, masterSystemTime=1691649039293 2023-08-10T06:30:39,322 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2579): Finished post open deploy task for ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343. 2023-08-10T06:30:39,322 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343. 2023-08-10T06:30:39,323 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=ab0a54958c666e251775ab33a9efc343, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:39,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=18, resume processing ppid=17 2023-08-10T06:30:39,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure ab0a54958c666e251775ab33a9efc343, server=jenkins-hbase3.apache.org,40209,1691649026642 in 186 msec 2023-08-10T06:30:39,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=17, resume processing ppid=16 2023-08-10T06:30:39,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16916490353991, region=ab0a54958c666e251775ab33a9efc343, ASSIGN in 346 msec 2023-08-10T06:30:39,332 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16916490353991 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:30:39,332 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16916490353991","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649039332"}]},"ts":"1691649039332"} 2023-08-10T06:30:39,334 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16916490353991, state=ENABLED in hbase:meta 2023-08-10T06:30:39,336 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16916490353991 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:30:39,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16916490353991 in 1.2090 sec 2023-08-10T06:30:40,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=16 2023-08-10T06:30:40,243 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns2:test-16916490353991, procId: 16 completed 2023-08-10T06:30:40,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40209 {}] regionserver.HRegion(8210): writing data to region ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343. with WAL disabled. Data may be lost in the event of a crash. 2023-08-10T06:30:40,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns3:test-16916490353992', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:30:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16916490353992 2023-08-10T06:30:40,334 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16916490353992 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:30:40,335 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:40,335 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16916490353992" procId is: 19 2023-08-10T06:30:40,336 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16916490353992 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:30:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:30:40,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:30:40,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:30:40,752 INFO [RegionOpenAndInit-ns3:test-16916490353992-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 66e77e1fb4a2e8bff5d5df89490b6752, NAME => 'ns3:test-16916490353992,,1691649040330.66e77e1fb4a2e8bff5d5df89490b6752.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16916490353992', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a 2023-08-10T06:30:40,774 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16916490353991' 2023-08-10T06:30:40,775 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1691649035399' 2023-08-10T06:30:40,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:30:41,167 DEBUG [RegionOpenAndInit-ns3:test-16916490353992-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16916490353992,,1691649040330.66e77e1fb4a2e8bff5d5df89490b6752.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:41,167 DEBUG [RegionOpenAndInit-ns3:test-16916490353992-pool-0 {}] regionserver.HRegion(1668): Closing 66e77e1fb4a2e8bff5d5df89490b6752, disabling compactions & flushes 2023-08-10T06:30:41,167 INFO [RegionOpenAndInit-ns3:test-16916490353992-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16916490353992,,1691649040330.66e77e1fb4a2e8bff5d5df89490b6752. 2023-08-10T06:30:41,167 DEBUG [RegionOpenAndInit-ns3:test-16916490353992-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16916490353992,,1691649040330.66e77e1fb4a2e8bff5d5df89490b6752. 2023-08-10T06:30:41,167 DEBUG [RegionOpenAndInit-ns3:test-16916490353992-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16916490353992,,1691649040330.66e77e1fb4a2e8bff5d5df89490b6752. after waiting 0 ms 2023-08-10T06:30:41,167 DEBUG [RegionOpenAndInit-ns3:test-16916490353992-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16916490353992,,1691649040330.66e77e1fb4a2e8bff5d5df89490b6752. 2023-08-10T06:30:41,167 INFO [RegionOpenAndInit-ns3:test-16916490353992-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16916490353992,,1691649040330.66e77e1fb4a2e8bff5d5df89490b6752. 2023-08-10T06:30:41,167 DEBUG [RegionOpenAndInit-ns3:test-16916490353992-pool-0 {}] regionserver.HRegion(1622): Region close journal for 66e77e1fb4a2e8bff5d5df89490b6752: 2023-08-10T06:30:41,169 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16916490353992 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:30:41,169 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16916490353992,,1691649040330.66e77e1fb4a2e8bff5d5df89490b6752.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1691649041169"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649041169"}]},"ts":"1691649041169"} 2023-08-10T06:30:41,171 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:30:41,172 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16916490353992 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:30:41,172 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16916490353992","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649041172"}]},"ts":"1691649041172"} 2023-08-10T06:30:41,174 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16916490353992, state=ENABLING in hbase:meta 2023-08-10T06:30:41,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16916490353992, region=66e77e1fb4a2e8bff5d5df89490b6752, ASSIGN}] 2023-08-10T06:30:41,180 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16916490353992, region=66e77e1fb4a2e8bff5d5df89490b6752, ASSIGN 2023-08-10T06:30:41,181 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16916490353992, region=66e77e1fb4a2e8bff5d5df89490b6752, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40209,1691649026642; forceNewPlan=false, retain=false 2023-08-10T06:30:41,333 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=66e77e1fb4a2e8bff5d5df89490b6752, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:41,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 66e77e1fb4a2e8bff5d5df89490b6752, server=jenkins-hbase3.apache.org,40209,1691649026642}] 2023-08-10T06:30:41,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:30:41,492 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16916490353992,,1691649040330.66e77e1fb4a2e8bff5d5df89490b6752. 2023-08-10T06:30:41,492 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 66e77e1fb4a2e8bff5d5df89490b6752, NAME => 'ns3:test-16916490353992,,1691649040330.66e77e1fb4a2e8bff5d5df89490b6752.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:30:41,493 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:30:41,493 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16916490353992 66e77e1fb4a2e8bff5d5df89490b6752 2023-08-10T06:30:41,493 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16916490353992,,1691649040330.66e77e1fb4a2e8bff5d5df89490b6752.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:41,493 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 66e77e1fb4a2e8bff5d5df89490b6752 2023-08-10T06:30:41,493 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 66e77e1fb4a2e8bff5d5df89490b6752 2023-08-10T06:30:41,495 INFO [StoreOpener-66e77e1fb4a2e8bff5d5df89490b6752-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 66e77e1fb4a2e8bff5d5df89490b6752 2023-08-10T06:30:41,497 INFO [StoreOpener-66e77e1fb4a2e8bff5d5df89490b6752-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66e77e1fb4a2e8bff5d5df89490b6752 columnFamilyName f 2023-08-10T06:30:41,497 DEBUG [StoreOpener-66e77e1fb4a2e8bff5d5df89490b6752-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:41,498 INFO [StoreOpener-66e77e1fb4a2e8bff5d5df89490b6752-1 {}] regionserver.HStore(324): Store=66e77e1fb4a2e8bff5d5df89490b6752/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:30:41,499 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns3/test-16916490353992/66e77e1fb4a2e8bff5d5df89490b6752 2023-08-10T06:30:41,499 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns3/test-16916490353992/66e77e1fb4a2e8bff5d5df89490b6752 2023-08-10T06:30:41,503 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 66e77e1fb4a2e8bff5d5df89490b6752 2023-08-10T06:30:41,505 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns3/test-16916490353992/66e77e1fb4a2e8bff5d5df89490b6752/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:30:41,506 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 66e77e1fb4a2e8bff5d5df89490b6752; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11338079680, jitterRate=0.05594095587730408}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:30:41,507 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 66e77e1fb4a2e8bff5d5df89490b6752: 2023-08-10T06:30:41,508 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2552): Post open deploy tasks for ns3:test-16916490353992,,1691649040330.66e77e1fb4a2e8bff5d5df89490b6752., pid=21, masterSystemTime=1691649041488 2023-08-10T06:30:41,510 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2579): Finished post open deploy task for ns3:test-16916490353992,,1691649040330.66e77e1fb4a2e8bff5d5df89490b6752. 2023-08-10T06:30:41,510 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16916490353992,,1691649040330.66e77e1fb4a2e8bff5d5df89490b6752. 2023-08-10T06:30:41,511 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=66e77e1fb4a2e8bff5d5df89490b6752, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:41,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=21, resume processing ppid=20 2023-08-10T06:30:41,516 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 66e77e1fb4a2e8bff5d5df89490b6752, server=jenkins-hbase3.apache.org,40209,1691649026642 in 178 msec 2023-08-10T06:30:41,518 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=20, resume processing ppid=19 2023-08-10T06:30:41,518 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16916490353992, region=66e77e1fb4a2e8bff5d5df89490b6752, ASSIGN in 338 msec 2023-08-10T06:30:41,519 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16916490353992 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:30:41,519 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16916490353992","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649041519"}]},"ts":"1691649041519"} 2023-08-10T06:30:41,521 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16916490353992, state=ENABLED in hbase:meta 2023-08-10T06:30:41,525 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16916490353992 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:30:41,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16916490353992 in 1.1940 sec 2023-08-10T06:30:42,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=19 2023-08-10T06:30:42,445 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns3:test-16916490353992, procId: 19 completed 2023-08-10T06:30:42,445 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16916490353992 get assigned. Timeout = 60000ms 2023-08-10T06:30:42,447 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-08-10T06:30:42,452 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16916490353992 assigned to meta. Checking AM states. 2023-08-10T06:30:42,452 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-08-10T06:30:42,452 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16916490353992 assigned. 2023-08-10T06:30:42,454 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'ns4:test-16916490353993', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:30:42,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16916490353993 2023-08-10T06:30:42,458 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16916490353993 execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:30:42,458 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:42,458 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16916490353993" procId is: 22 2023-08-10T06:30:42,459 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16916490353993 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:30:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:30:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:30:42,591 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:30:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:30:42,873 INFO [RegionOpenAndInit-ns4:test-16916490353993-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 22e2ac566be23eb7afc4a9349ef3e613, NAME => 'ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16916490353993', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a 2023-08-10T06:30:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:30:43,287 DEBUG [RegionOpenAndInit-ns4:test-16916490353993-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:43,287 DEBUG [RegionOpenAndInit-ns4:test-16916490353993-pool-0 {}] regionserver.HRegion(1668): Closing 22e2ac566be23eb7afc4a9349ef3e613, disabling compactions & flushes 2023-08-10T06:30:43,287 INFO [RegionOpenAndInit-ns4:test-16916490353993-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613. 2023-08-10T06:30:43,287 DEBUG [RegionOpenAndInit-ns4:test-16916490353993-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613. 2023-08-10T06:30:43,287 DEBUG [RegionOpenAndInit-ns4:test-16916490353993-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613. after waiting 0 ms 2023-08-10T06:30:43,287 DEBUG [RegionOpenAndInit-ns4:test-16916490353993-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613. 2023-08-10T06:30:43,287 INFO [RegionOpenAndInit-ns4:test-16916490353993-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613. 2023-08-10T06:30:43,287 DEBUG [RegionOpenAndInit-ns4:test-16916490353993-pool-0 {}] regionserver.HRegion(1622): Region close journal for 22e2ac566be23eb7afc4a9349ef3e613: 2023-08-10T06:30:43,289 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16916490353993 execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:30:43,289 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1691649043289"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649043289"}]},"ts":"1691649043289"} 2023-08-10T06:30:43,291 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:30:43,292 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16916490353993 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:30:43,293 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16916490353993","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649043292"}]},"ts":"1691649043292"} 2023-08-10T06:30:43,294 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16916490353993, state=ENABLING in hbase:meta 2023-08-10T06:30:43,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16916490353993, region=22e2ac566be23eb7afc4a9349ef3e613, ASSIGN}] 2023-08-10T06:30:43,300 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16916490353993, region=22e2ac566be23eb7afc4a9349ef3e613, ASSIGN 2023-08-10T06:30:43,301 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16916490353993, region=22e2ac566be23eb7afc4a9349ef3e613, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40209,1691649026642; forceNewPlan=false, retain=false 2023-08-10T06:30:43,453 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=22e2ac566be23eb7afc4a9349ef3e613, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:43,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 22e2ac566be23eb7afc4a9349ef3e613, server=jenkins-hbase3.apache.org,40209,1691649026642}] 2023-08-10T06:30:43,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:30:43,612 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613. 2023-08-10T06:30:43,612 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 22e2ac566be23eb7afc4a9349ef3e613, NAME => 'ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:30:43,613 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:30:43,613 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16916490353993 22e2ac566be23eb7afc4a9349ef3e613 2023-08-10T06:30:43,613 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:43,613 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 22e2ac566be23eb7afc4a9349ef3e613 2023-08-10T06:30:43,613 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 22e2ac566be23eb7afc4a9349ef3e613 2023-08-10T06:30:43,615 INFO [StoreOpener-22e2ac566be23eb7afc4a9349ef3e613-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 22e2ac566be23eb7afc4a9349ef3e613 2023-08-10T06:30:43,617 INFO [StoreOpener-22e2ac566be23eb7afc4a9349ef3e613-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22e2ac566be23eb7afc4a9349ef3e613 columnFamilyName f 2023-08-10T06:30:43,617 DEBUG [StoreOpener-22e2ac566be23eb7afc4a9349ef3e613-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:43,617 INFO [StoreOpener-22e2ac566be23eb7afc4a9349ef3e613-1 {}] regionserver.HStore(324): Store=22e2ac566be23eb7afc4a9349ef3e613/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:30:43,618 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns4/test-16916490353993/22e2ac566be23eb7afc4a9349ef3e613 2023-08-10T06:30:43,619 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns4/test-16916490353993/22e2ac566be23eb7afc4a9349ef3e613 2023-08-10T06:30:43,622 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 22e2ac566be23eb7afc4a9349ef3e613 2023-08-10T06:30:43,625 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns4/test-16916490353993/22e2ac566be23eb7afc4a9349ef3e613/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:30:43,626 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 22e2ac566be23eb7afc4a9349ef3e613; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11360362720, jitterRate=0.058016225695610046}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-08-10T06:30:43,626 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 22e2ac566be23eb7afc4a9349ef3e613: 2023-08-10T06:30:43,627 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2552): Post open deploy tasks for ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613., pid=24, masterSystemTime=1691649043608 2023-08-10T06:30:43,629 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2579): Finished post open deploy task for ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613. 2023-08-10T06:30:43,629 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613. 2023-08-10T06:30:43,630 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=22e2ac566be23eb7afc4a9349ef3e613, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:43,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=24, resume processing ppid=23 2023-08-10T06:30:43,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 22e2ac566be23eb7afc4a9349ef3e613, server=jenkins-hbase3.apache.org,40209,1691649026642 in 177 msec 2023-08-10T06:30:43,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=23, resume processing ppid=22 2023-08-10T06:30:43,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16916490353993, region=22e2ac566be23eb7afc4a9349ef3e613, ASSIGN in 337 msec 2023-08-10T06:30:43,639 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16916490353993 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:30:43,639 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16916490353993","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649043639"}]},"ts":"1691649043639"} 2023-08-10T06:30:43,641 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16916490353993, state=ENABLED in hbase:meta 2023-08-10T06:30:43,643 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16916490353993 execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:30:43,645 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16916490353993 in 1.1890 sec 2023-08-10T06:30:43,812 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16916490353991 2023-08-10T06:30:43,812 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16916490353991 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:30:43,813 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1691649035399 2023-08-10T06:30:43,813 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1691649035399 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:30:43,814 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16916490353993 2023-08-10T06:30:43,814 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16916490353993 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:30:43,815 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16916490353992 2023-08-10T06:30:43,815 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16916490353992 Metrics about Tables on a single HBase RegionServer 2023-08-10T06:30:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=22 2023-08-10T06:30:44,568 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: ns4:test-16916490353993, procId: 22 completed 2023-08-10T06:30:44,568 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16916490353993 get assigned. Timeout = 60000ms 2023-08-10T06:30:44,568 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-08-10T06:30:44,572 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16916490353993 assigned to meta. Checking AM states. 2023-08-10T06:30:44,573 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-08-10T06:30:44,573 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16916490353993 assigned. 2023-08-10T06:30:44,573 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x36539b84 to 127.0.0.1:55224 2023-08-10T06:30:44,573 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:44,613 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] hbase.ResourceChecker(147): before: backup.TestIncrementalBackupWithFailures#testIncBackupRestore[0] Thread=586, OpenFileDescriptor=667, MaxFileDescriptor=60000, SystemLoadAverage=149, ProcessCount=171, AvailableMemoryMB=1793 2023-08-10T06:30:44,613 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] hbase.ResourceChecker(130): Thread=586 is superior to 500 2023-08-10T06:30:44,616 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] backup.TestIncrementalBackupWithFailures(82): create full backup image for all tables 2023-08-10T06:30:44,623 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster$13(2699): Client=jenkins//172.31.12.81 modify table test-1691649035399 from 'test-1691649035399', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'test-1691649035399', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:30:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=25, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=test-1691649035399 2023-08-10T06:30:45,060 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/.tabledesc/.tableinfo.0000000001.367 2023-08-10T06:30:45,060 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/.tabledesc/.tableinfo.0000000002.598 2023-08-10T06:30:45,066 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=test-1691649035399}] 2023-08-10T06:30:45,082 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1691649035399, region=41ac6ed181c2526bbcdb3ef2c04e3054, REOPEN/MOVE}] 2023-08-10T06:30:45,083 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1691649035399, region=41ac6ed181c2526bbcdb3ef2c04e3054, REOPEN/MOVE 2023-08-10T06:30:45,083 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=41ac6ed181c2526bbcdb3ef2c04e3054, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:45,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; CloseRegionProcedure 41ac6ed181c2526bbcdb3ef2c04e3054, server=jenkins-hbase3.apache.org,40209,1691649026642}] 2023-08-10T06:30:45,246 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(119): Close 41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:30:45,247 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1668): Closing 41ac6ed181c2526bbcdb3ef2c04e3054, disabling compactions & flushes 2023-08-10T06:30:45,247 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1690): Closing region test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:30:45,247 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:30:45,247 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1778): Acquired close lock on test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. after waiting 0 ms 2023-08-10T06:30:45,247 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1788): Updates disabled for region test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:30:45,248 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2819): Flushing 41ac6ed181c2526bbcdb3ef2c04e3054 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-08-10T06:30:45,329 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/.tmp/f/fa1d8c8d9a7f448881583cf4837e9f5e is 37, key is row10/f:q1/1691649038040/Put/seqid=0 2023-08-10T06:30:45,348 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/.tmp/f/fa1d8c8d9a7f448881583cf4837e9f5e 2023-08-10T06:30:45,424 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: fa1d8c8d9a7f448881583cf4837e9f5e 2023-08-10T06:30:45,428 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/.tmp/f/fa1d8c8d9a7f448881583cf4837e9f5e as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/f/fa1d8c8d9a7f448881583cf4837e9f5e 2023-08-10T06:30:45,437 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/f/fa1d8c8d9a7f448881583cf4837e9f5e, entries=99, sequenceid=103, filesize=8.2 K 2023-08-10T06:30:45,441 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 41ac6ed181c2526bbcdb3ef2c04e3054 in 193ms, sequenceid=103, compaction requested=false 2023-08-10T06:30:45,445 DEBUG [StoreFileCloser-41ac6ed181c2526bbcdb3ef2c04e3054-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: fa1d8c8d9a7f448881583cf4837e9f5e 2023-08-10T06:30:45,451 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-08-10T06:30:45,452 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:30:45,453 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1909): Closed test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:30:45,453 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1622): Region close journal for 41ac6ed181c2526bbcdb3ef2c04e3054: 2023-08-10T06:30:45,454 WARN [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionServer(3731): Not adding moved region record: 41ac6ed181c2526bbcdb3ef2c04e3054 to self. 2023-08-10T06:30:45,456 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(163): Closed 41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:30:45,457 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=41ac6ed181c2526bbcdb3ef2c04e3054, regionState=CLOSED 2023-08-10T06:30:45,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=28, resume processing ppid=27 2023-08-10T06:30:45,461 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=28, ppid=27, state=SUCCESS; CloseRegionProcedure 41ac6ed181c2526bbcdb3ef2c04e3054, server=jenkins-hbase3.apache.org,40209,1691649026642 in 373 msec 2023-08-10T06:30:45,462 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1691649035399, region=41ac6ed181c2526bbcdb3ef2c04e3054, REOPEN/MOVE; state=CLOSED, location=jenkins-hbase3.apache.org,40209,1691649026642; forceNewPlan=false, retain=true 2023-08-10T06:30:45,613 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=41ac6ed181c2526bbcdb3ef2c04e3054, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:45,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=29, ppid=27, state=RUNNABLE; OpenRegionProcedure 41ac6ed181c2526bbcdb3ef2c04e3054, server=jenkins-hbase3.apache.org,40209,1691649026642}] 2023-08-10T06:30:45,773 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(130): Open test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:30:45,773 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7254): Opening region: {ENCODED => 41ac6ed181c2526bbcdb3ef2c04e3054, NAME => 'test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:30:45,774 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:30:45,774 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1691649035399 41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:30:45,774 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(888): Instantiated test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:45,774 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7296): checking encryption for 41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:30:45,774 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(7299): checking classloading for 41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:30:45,776 INFO [StoreOpener-41ac6ed181c2526bbcdb3ef2c04e3054-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:30:45,779 INFO [StoreOpener-41ac6ed181c2526bbcdb3ef2c04e3054-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41ac6ed181c2526bbcdb3ef2c04e3054 columnFamilyName f 2023-08-10T06:30:45,779 DEBUG [StoreOpener-41ac6ed181c2526bbcdb3ef2c04e3054-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:45,791 DEBUG [StoreOpener-41ac6ed181c2526bbcdb3ef2c04e3054-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/f/fa1d8c8d9a7f448881583cf4837e9f5e 2023-08-10T06:30:45,791 INFO [StoreOpener-41ac6ed181c2526bbcdb3ef2c04e3054-1 {}] regionserver.HStore(324): Store=41ac6ed181c2526bbcdb3ef2c04e3054/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:30:45,791 INFO [StoreOpener-41ac6ed181c2526bbcdb3ef2c04e3054-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f3 of region 41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:30:45,793 INFO [StoreOpener-41ac6ed181c2526bbcdb3ef2c04e3054-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41ac6ed181c2526bbcdb3ef2c04e3054 columnFamilyName f3 2023-08-10T06:30:45,793 DEBUG [StoreOpener-41ac6ed181c2526bbcdb3ef2c04e3054-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:45,794 INFO [StoreOpener-41ac6ed181c2526bbcdb3ef2c04e3054-1 {}] regionserver.HStore(324): Store=41ac6ed181c2526bbcdb3ef2c04e3054/f3, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:30:45,795 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:30:45,797 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:30:45,799 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test-1691649035399 descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:30:45,801 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1076): writing seq id for 41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:30:45,803 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(1093): Opened 41ac6ed181c2526bbcdb3ef2c04e3054; next sequenceid=107; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11465577600, jitterRate=0.06781512498855591}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:30:45,804 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegion(992): Region open journal for 41ac6ed181c2526bbcdb3ef2c04e3054: 2023-08-10T06:30:45,805 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2552): Post open deploy tasks for test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054., pid=29, masterSystemTime=1691649045767 2023-08-10T06:30:45,807 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] regionserver.HRegionServer(2579): Finished post open deploy task for test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:30:45,808 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=29}] handler.AssignRegionHandler(158): Opened test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:30:45,808 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=41ac6ed181c2526bbcdb3ef2c04e3054, regionState=OPEN, openSeqNum=107, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:45,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=29, resume processing ppid=27 2023-08-10T06:30:45,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=29, ppid=27, state=SUCCESS; OpenRegionProcedure 41ac6ed181c2526bbcdb3ef2c04e3054, server=jenkins-hbase3.apache.org,40209,1691649026642 in 196 msec 2023-08-10T06:30:45,816 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=27, resume processing ppid=26 2023-08-10T06:30:45,816 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=test-1691649035399, region=41ac6ed181c2526bbcdb3ef2c04e3054, REOPEN/MOVE in 732 msec 2023-08-10T06:30:45,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=26, resume processing ppid=25 2023-08-10T06:30:45,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=26, ppid=25, state=SUCCESS; ReopenTableRegionsProcedure table=test-1691649035399 in 753 msec 2023-08-10T06:30:45,822 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=25, state=SUCCESS; ModifyTableProcedure table=test-1691649035399 in 1.1930 sec 2023-08-10T06:30:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=25 2023-08-10T06:30:45,834 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x20ace7d2 to 127.0.0.1:55224 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:30:45,840 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3570a87c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:45,842 DEBUG [hconnection-0x3d40ce37-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:45,844 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57904, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:45,878 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:30:45,879 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37724, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:30:45,889 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster$17(3366): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-08-10T06:30:45,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=30, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-08-10T06:30:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=30 2023-08-10T06:30:45,905 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-08-10T06:30:45,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=30, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 19 msec 2023-08-10T06:30:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=30 2023-08-10T06:30:46,010 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:30:46,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-08-10T06:30:46,015 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:30:46,015 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:46,015 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 31 2023-08-10T06:30:46,017 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:30:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=31 2023-08-10T06:30:46,030 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 4cb36c501bd471befb9ec81ac2da937f, NAME => 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a 2023-08-10T06:30:46,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=31 2023-08-10T06:30:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=31 2023-08-10T06:30:46,441 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:46,441 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 4cb36c501bd471befb9ec81ac2da937f, disabling compactions & flushes 2023-08-10T06:30:46,441 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:30:46,441 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:30:46,441 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. after waiting 0 ms 2023-08-10T06:30:46,441 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:30:46,441 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:30:46,441 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 4cb36c501bd471befb9ec81ac2da937f: 2023-08-10T06:30:46,443 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:30:46,443 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1691649046443"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649046443"}]},"ts":"1691649046443"} 2023-08-10T06:30:46,445 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:30:46,446 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:30:46,446 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649046446"}]},"ts":"1691649046446"} 2023-08-10T06:30:46,448 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-08-10T06:30:46,452 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, ASSIGN}] 2023-08-10T06:30:46,454 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, ASSIGN 2023-08-10T06:30:46,455 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=32, ppid=31, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40209,1691649026642; forceNewPlan=false, retain=false 2023-08-10T06:30:46,607 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=32 updating hbase:meta row=4cb36c501bd471befb9ec81ac2da937f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:46,609 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; OpenRegionProcedure 4cb36c501bd471befb9ec81ac2da937f, server=jenkins-hbase3.apache.org,40209,1691649026642}] 2023-08-10T06:30:46,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=31 2023-08-10T06:30:46,766 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(130): Open backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:30:46,766 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7254): Opening region: {ENCODED => 4cb36c501bd471befb9ec81ac2da937f, NAME => 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:30:46,767 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:30:46,767 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:30:46,767 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(888): Instantiated backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:46,767 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7296): checking encryption for 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:30:46,767 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7299): checking classloading for 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:30:46,769 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:30:46,771 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4cb36c501bd471befb9ec81ac2da937f columnFamilyName meta 2023-08-10T06:30:46,771 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:46,772 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.HStore(324): Store=4cb36c501bd471befb9ec81ac2da937f/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:30:46,772 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:30:46,774 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4cb36c501bd471befb9ec81ac2da937f columnFamilyName session 2023-08-10T06:30:46,774 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:46,774 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.HStore(324): Store=4cb36c501bd471befb9ec81ac2da937f/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:30:46,775 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:30:46,776 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16916490353993' 2023-08-10T06:30:46,776 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:30:46,777 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16916490353992' 2023-08-10T06:30:46,778 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:30:46,781 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1076): writing seq id for 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:30:46,784 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:30:46,784 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): Opened 4cb36c501bd471befb9ec81ac2da937f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11699099360, jitterRate=0.0895635336637497}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:30:46,785 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(992): Region open journal for 4cb36c501bd471befb9ec81ac2da937f: 2023-08-10T06:30:46,786 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f., pid=33, masterSystemTime=1691649046761 2023-08-10T06:30:46,788 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:30:46,788 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(158): Opened backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:30:46,789 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=32 updating hbase:meta row=4cb36c501bd471befb9ec81ac2da937f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:46,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=33, resume processing ppid=32 2023-08-10T06:30:46,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=33, ppid=32, state=SUCCESS; OpenRegionProcedure 4cb36c501bd471befb9ec81ac2da937f, server=jenkins-hbase3.apache.org,40209,1691649026642 in 182 msec 2023-08-10T06:30:46,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=32, resume processing ppid=31 2023-08-10T06:30:46,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=32, ppid=31, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, ASSIGN in 341 msec 2023-08-10T06:30:46,796 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:30:46,796 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649046796"}]},"ts":"1691649046796"} 2023-08-10T06:30:46,798 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-08-10T06:30:46,800 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=31, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:30:46,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=31, state=SUCCESS; CreateTableProcedure table=backup:system in 790 msec 2023-08-10T06:30:47,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=31 2023-08-10T06:30:47,123 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system, procId: 31 completed 2023-08-10T06:30:47,137 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster$4(2331): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:30:47,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-08-10T06:30:47,141 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-08-10T06:30:47,141 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:47,141 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(708): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 34 2023-08-10T06:30:47,142 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-08-10T06:30:47,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:30:47,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:30:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:30:47,558 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => e4d21e17fa5b460ebe5cf90f2eb64bca, NAME => 'backup:system_bulk,,1691649047137.e4d21e17fa5b460ebe5cf90f2eb64bca.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a 2023-08-10T06:30:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:30:47,971 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1691649047137.e4d21e17fa5b460ebe5cf90f2eb64bca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:47,971 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing e4d21e17fa5b460ebe5cf90f2eb64bca, disabling compactions & flushes 2023-08-10T06:30:47,971 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1691649047137.e4d21e17fa5b460ebe5cf90f2eb64bca. 2023-08-10T06:30:47,971 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1691649047137.e4d21e17fa5b460ebe5cf90f2eb64bca. 2023-08-10T06:30:47,971 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1691649047137.e4d21e17fa5b460ebe5cf90f2eb64bca. after waiting 0 ms 2023-08-10T06:30:47,971 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1691649047137.e4d21e17fa5b460ebe5cf90f2eb64bca. 2023-08-10T06:30:47,971 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1691649047137.e4d21e17fa5b460ebe5cf90f2eb64bca. 2023-08-10T06:30:47,971 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for e4d21e17fa5b460ebe5cf90f2eb64bca: 2023-08-10T06:30:47,973 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-08-10T06:30:47,973 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1691649047137.e4d21e17fa5b460ebe5cf90f2eb64bca.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1691649047973"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649047973"}]},"ts":"1691649047973"} 2023-08-10T06:30:47,975 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:30:47,976 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-08-10T06:30:47,976 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649047976"}]},"ts":"1691649047976"} 2023-08-10T06:30:47,978 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-08-10T06:30:47,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=e4d21e17fa5b460ebe5cf90f2eb64bca, ASSIGN}] 2023-08-10T06:30:47,985 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=e4d21e17fa5b460ebe5cf90f2eb64bca, ASSIGN 2023-08-10T06:30:47,986 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=e4d21e17fa5b460ebe5cf90f2eb64bca, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,40209,1691649026642; forceNewPlan=false, retain=false 2023-08-10T06:30:48,137 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=e4d21e17fa5b460ebe5cf90f2eb64bca, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:48,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE; OpenRegionProcedure e4d21e17fa5b460ebe5cf90f2eb64bca, server=jenkins-hbase3.apache.org,40209,1691649026642}] 2023-08-10T06:30:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:30:48,296 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1691649047137.e4d21e17fa5b460ebe5cf90f2eb64bca. 2023-08-10T06:30:48,296 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7254): Opening region: {ENCODED => e4d21e17fa5b460ebe5cf90f2eb64bca, NAME => 'backup:system_bulk,,1691649047137.e4d21e17fa5b460ebe5cf90f2eb64bca.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:30:48,297 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:30:48,297 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk e4d21e17fa5b460ebe5cf90f2eb64bca 2023-08-10T06:30:48,297 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1691649047137.e4d21e17fa5b460ebe5cf90f2eb64bca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:30:48,297 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7296): checking encryption for e4d21e17fa5b460ebe5cf90f2eb64bca 2023-08-10T06:30:48,297 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7299): checking classloading for e4d21e17fa5b460ebe5cf90f2eb64bca 2023-08-10T06:30:48,299 INFO [StoreOpener-e4d21e17fa5b460ebe5cf90f2eb64bca-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region e4d21e17fa5b460ebe5cf90f2eb64bca 2023-08-10T06:30:48,300 INFO [StoreOpener-e4d21e17fa5b460ebe5cf90f2eb64bca-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4d21e17fa5b460ebe5cf90f2eb64bca columnFamilyName meta 2023-08-10T06:30:48,300 DEBUG [StoreOpener-e4d21e17fa5b460ebe5cf90f2eb64bca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:48,301 INFO [StoreOpener-e4d21e17fa5b460ebe5cf90f2eb64bca-1 {}] regionserver.HStore(324): Store=e4d21e17fa5b460ebe5cf90f2eb64bca/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:30:48,301 INFO [StoreOpener-e4d21e17fa5b460ebe5cf90f2eb64bca-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region e4d21e17fa5b460ebe5cf90f2eb64bca 2023-08-10T06:30:48,303 INFO [StoreOpener-e4d21e17fa5b460ebe5cf90f2eb64bca-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4d21e17fa5b460ebe5cf90f2eb64bca columnFamilyName session 2023-08-10T06:30:48,303 DEBUG [StoreOpener-e4d21e17fa5b460ebe5cf90f2eb64bca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:30:48,304 INFO [StoreOpener-e4d21e17fa5b460ebe5cf90f2eb64bca-1 {}] regionserver.HStore(324): Store=e4d21e17fa5b460ebe5cf90f2eb64bca/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:30:48,304 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system_bulk/e4d21e17fa5b460ebe5cf90f2eb64bca 2023-08-10T06:30:48,305 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system_bulk/e4d21e17fa5b460ebe5cf90f2eb64bca 2023-08-10T06:30:48,307 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:30:48,308 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1076): writing seq id for e4d21e17fa5b460ebe5cf90f2eb64bca 2023-08-10T06:30:48,311 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system_bulk/e4d21e17fa5b460ebe5cf90f2eb64bca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-08-10T06:30:48,312 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): Opened e4d21e17fa5b460ebe5cf90f2eb64bca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11998698880, jitterRate=0.11746591329574585}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:30:48,312 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(992): Region open journal for e4d21e17fa5b460ebe5cf90f2eb64bca: 2023-08-10T06:30:48,313 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system_bulk,,1691649047137.e4d21e17fa5b460ebe5cf90f2eb64bca., pid=36, masterSystemTime=1691649048292 2023-08-10T06:30:48,315 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system_bulk,,1691649047137.e4d21e17fa5b460ebe5cf90f2eb64bca. 2023-08-10T06:30:48,315 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1691649047137.e4d21e17fa5b460ebe5cf90f2eb64bca. 2023-08-10T06:30:48,316 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=e4d21e17fa5b460ebe5cf90f2eb64bca, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:48,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=36, resume processing ppid=35 2023-08-10T06:30:48,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=36, ppid=35, state=SUCCESS; OpenRegionProcedure e4d21e17fa5b460ebe5cf90f2eb64bca, server=jenkins-hbase3.apache.org,40209,1691649026642 in 178 msec 2023-08-10T06:30:48,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=35, resume processing ppid=34 2023-08-10T06:30:48,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=35, ppid=34, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=e4d21e17fa5b460ebe5cf90f2eb64bca, ASSIGN in 337 msec 2023-08-10T06:30:48,323 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-08-10T06:30:48,323 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649048323"}]},"ts":"1691649048323"} 2023-08-10T06:30:48,325 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-08-10T06:30:48,328 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-08-10T06:30:48,330 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=34, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 1.1910 sec 2023-08-10T06:30:49,116 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:30:49,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=34 2023-08-10T06:30:49,249 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] client.HBaseAdmin$TableFuture(3590): Operation: CREATE, Table Name: backup:system_bulk, procId: 34 completed 2023-08-10T06:30:49,260 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:30:49,305 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:30:49,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649049305 (current time:1691649049305). 2023-08-10T06:30:49,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:30:49,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:30:49,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6cc11cb4 to 127.0.0.1:55224 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:30:49,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e668ffe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:49,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:49,319 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57918, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:49,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6cc11cb4 to 127.0.0.1:55224 2023-08-10T06:30:49,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:49,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:30:49,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=37, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:30:49,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-08-10T06:30:49,335 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:30:49,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-08-10T06:30:49,340 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:30:49,343 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:30:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-08-10T06:30:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-08-10T06:30:49,755 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:30:49,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; SnapshotRegionProcedure 4cb36c501bd471befb9ec81ac2da937f}] 2023-08-10T06:30:49,765 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=38, ppid=37, state=RUNNABLE; SnapshotRegionProcedure 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:30:49,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40209 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=38 2023-08-10T06:30:49,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:30:49,929 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HRegion(2819): Flushing 4cb36c501bd471befb9ec81ac2da937f 2/2 column families, dataSize=45 B heapSize=632 B 2023-08-10T06:30:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-08-10T06:30:49,944 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/34d64473585d4371aa81253c6048c0d8 is 49, key is activesession:/session:c/1691649049291/Put/seqid=0 2023-08-10T06:30:49,955 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/34d64473585d4371aa81253c6048c0d8 2023-08-10T06:30:49,962 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 34d64473585d4371aa81253c6048c0d8 2023-08-10T06:30:49,964 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/34d64473585d4371aa81253c6048c0d8 as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/34d64473585d4371aa81253c6048c0d8 2023-08-10T06:30:49,975 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/34d64473585d4371aa81253c6048c0d8, entries=1, sequenceid=5, filesize=4.9 K 2023-08-10T06:30:49,978 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 4cb36c501bd471befb9ec81ac2da937f in 48ms, sequenceid=5, compaction requested=false 2023-08-10T06:30:49,978 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-08-10T06:30:49,979 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.HRegion(2520): Flush status journal for 4cb36c501bd471befb9ec81ac2da937f: 2023-08-10T06:30:49,979 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. for snapshot_backup_system completed. 2023-08-10T06:30:49,980 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:30:49,990 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:30:49,994 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-08-10T06:30:49,995 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/34d64473585d4371aa81253c6048c0d8] hfiles 2023-08-10T06:30:49,995 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/34d64473585d4371aa81253c6048c0d8 for snapshot=snapshot_backup_system 2023-08-10T06:30:50,412 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:30:50,416 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=38}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=38 2023-08-10T06:30:50,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster(3994): Remote procedure done, pid=38 2023-08-10T06:30:50,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:30:50,420 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=38, ppid=37, state=RUNNABLE; SnapshotRegionProcedure 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:30:50,423 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=38, resume processing ppid=37 2023-08-10T06:30:50,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=38, ppid=37, state=SUCCESS; SnapshotRegionProcedure 4cb36c501bd471befb9ec81ac2da937f in 663 msec 2023-08-10T06:30:50,424 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:30:50,425 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:30:50,428 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:30:50,429 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:30:50,431 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:30:50,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-08-10T06:30:50,855 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:30:50,868 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:30:50,869 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:30:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-08-10T06:30:51,699 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:30:51,699 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-08-10T06:30:51,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=37, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.3710 sec 2023-08-10T06:30:52,778 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-08-10T06:30:53,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=37 2023-08-10T06:30:53,446 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: backup:system, procId: 37 completed 2023-08-10T06:30:53,446 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(115): Backup backup_1691649045860 started at 1691649053446. 2023-08-10T06:30:53,467 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(119): Backup session backup_1691649045860 has been started. 2023-08-10T06:30:53,470 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-08-10T06:30:53,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:30:53,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:30:53,487 INFO [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:30:53,487 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:30:53,488 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:30:53,488 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:30:53,488 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:30:53,488 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:30:53,490 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:53,490 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:30:53,490 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:30:53,490 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:30:53,490 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:53,490 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:30:53,490 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:30:53,491 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-08-10T06:30:53,491 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:30:53,491 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:30:53,491 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:30:53,493 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:30:53,493 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:30:53,493 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:30:53,493 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:30:53,494 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:30:53,494 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:30:53,494 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,40209,1691649026642' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:30:53,496 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:53,496 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:30:53,496 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:53,497 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:30:53,497 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:30:53,497 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-08-10T06:30:53,497 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:30:53,497 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:30:53,497 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:30:53,498 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:30:53,498 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:53,498 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:30:53,499 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,40209,1691649026642' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:30:53,499 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:30:53,499 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@13c8c42c[Count = 0] remaining members to acquire global barrier 2023-08-10T06:30:53,499 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:30:53,504 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:30:53,504 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:30:53,504 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:30:53,504 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:30:53,504 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:53,504 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:30:53,505 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:30:53,505 DEBUG [rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:53,505 INFO [rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691649028773 highest: 1691649030774 on jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:53,505 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1:(num 1691649030774) roll requested 2023-08-10T06:30:53,517 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506, exclude list is [], retry=0 2023-08-10T06:30:53,521 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK] 2023-08-10T06:30:53,524 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649030774 with entries=11, filesize=2.04 KB; new WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 2023-08-10T06:30:53,525 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK]] 2023-08-10T06:30:53,525 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649030774 is not closed yet, will try archiving it next time 2023-08-10T06:30:53,525 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta:.meta(num 1691649029122) roll requested 2023-08-10T06:30:53,537 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649053525.meta, exclude list is [], retry=0 2023-08-10T06:30:53,541 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK] 2023-08-10T06:30:53,543 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649029122.meta with entries=40, filesize=12.94 KB; new WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649053525.meta 2023-08-10T06:30:53,543 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK]] 2023-08-10T06:30:53,543 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649029122.meta is not closed yet, will try archiving it next time 2023-08-10T06:30:53,543 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0:(num 1691649028773) roll requested 2023-08-10T06:30:53,556 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543, exclude list is [], retry=0 2023-08-10T06:30:53,560 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK] 2023-08-10T06:30:53,563 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649028773 with entries=18, filesize=3.96 KB; new WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543 2023-08-10T06:30:53,564 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK]] 2023-08-10T06:30:53,564 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649028773 is not closed yet, will try archiving it next time 2023-08-10T06:30:53,566 DEBUG [rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-08-10T06:30:53,566 INFO [rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691649053543 on jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:53,567 DEBUG [rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:30:53,568 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33762, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-08-10T06:30:53,587 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:30:53,587 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,40209,1691649026642' in zk 2023-08-10T06:30:53,589 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:53,589 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:30:53,589 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:53,589 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:30:53,590 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:30:53,590 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:30:53,590 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:30:53,591 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:30:53,591 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:30:53,591 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:30:53,592 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:53,592 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:30:53,592 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:30:53,593 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:53,593 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,40209,1691649026642': 2023-08-10T06:30:53,593 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,40209,1691649026642' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:30:53,593 INFO [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:30:53,593 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:30:53,594 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:30:53,594 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:30:53,594 INFO [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:30:53,596 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:30:53,596 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:30:53,596 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:30:53,596 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:30:53,596 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:30:53,596 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:30:53,596 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:30:53,596 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:30:53,596 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:30:53,597 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:30:53,597 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:30:53,597 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:53,597 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:30:53,597 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:30:53,598 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:30:53,598 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:30:53,598 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:53,598 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:53,599 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:30:53,599 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:30:53,600 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:53,606 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:53,606 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:30:53,606 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-08-10T06:30:53,606 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-08-10T06:30:53,606 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:30:53,606 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): regionserver:40209-0x10060ecfb4e0001, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-08-10T06:30:53,606 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:30:53,606 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:30:53,606 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:30:53,606 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-08-10T06:30:53,607 DEBUG [Listener at localhost/39735-EventThread {}] zookeeper.ZKWatcher(604): master:46449-0x10060ecfb4e0000, quorum=127.0.0.1:55224, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-08-10T06:30:53,607 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:30:53,607 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:30:53,608 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:30:53,608 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:30:53,608 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:30:53,610 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] client.HBaseAdmin(2751): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-08-10T06:30:53,610 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] client.HBaseAdmin(2759): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-08-10T06:30:53,710 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] client.HBaseAdmin(2765): Getting current status of procedure from master... 2023-08-10T06:30:53,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:30:53,721 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691649053720_default_test-1691649035399 table=test-1691649035399 type=FLUSH ttl=0 } 2023-08-10T06:30:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649053721 (current time:1691649053721). 2023-08-10T06:30:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:30:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691649053720_default_test-1691649035399 VERSION not specified, setting to 2 2023-08-10T06:30:53,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5d04cb81 to 127.0.0.1:55224 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:30:53,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15e7f27c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:53,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:53,732 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:50864, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5d04cb81 to 127.0.0.1:55224 2023-08-10T06:30:53,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:53,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:30:53,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=39, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1691649053720_default_test-1691649035399 table=test-1691649035399 type=FLUSH ttl=0 } 2023-08-10T06:30:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691649053720_default_test-1691649035399 table=test-1691649035399 type=FLUSH ttl=0 }, snapshot procedure id = 39 2023-08-10T06:30:53,736 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1691649053720_default_test-1691649035399 table=test-1691649035399 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:30:53,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-08-10T06:30:53,737 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1691649053720_default_test-1691649035399 table=test-1691649035399 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:30:53,740 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1691649053720_default_test-1691649035399 table=test-1691649035399 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:30:53,812 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-08-10T06:30:53,812 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-08-10T06:30:53,813 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-08-10T06:30:53,813 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-08-10T06:30:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-08-10T06:30:54,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-08-10T06:30:54,151 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1691649053720_default_test-1691649035399 table=test-1691649035399 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:30:54,151 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE; SnapshotRegionProcedure 41ac6ed181c2526bbcdb3ef2c04e3054}] 2023-08-10T06:30:54,153 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE; SnapshotRegionProcedure 41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:30:54,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40209 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=40 2023-08-10T06:30:54,305 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:30:54,306 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2819): Flushing 41ac6ed181c2526bbcdb3ef2c04e3054 2/2 column families, dataSize=324 B heapSize=1.25 KB 2023-08-10T06:30:54,322 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/.tmp/f3/469af5dd68d44838a76a85d58d415d3a is 58, key is row-test-1691649035399-3-0/f3:q1/1691649045848/Put/seqid=0 2023-08-10T06:30:54,333 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=324 B at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/.tmp/f3/469af5dd68d44838a76a85d58d415d3a 2023-08-10T06:30:54,340 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 469af5dd68d44838a76a85d58d415d3a 2023-08-10T06:30:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-08-10T06:30:54,341 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/.tmp/f3/469af5dd68d44838a76a85d58d415d3a as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/f3/469af5dd68d44838a76a85d58d415d3a 2023-08-10T06:30:54,348 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/f3/469af5dd68d44838a76a85d58d415d3a, entries=6, sequenceid=115, filesize=5.2 K 2023-08-10T06:30:54,350 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(3022): Finished flush of dataSize ~324 B/324, heapSize ~1008 B/1008, currentSize=0 B/0 for 41ac6ed181c2526bbcdb3ef2c04e3054 in 44ms, sequenceid=115, compaction requested=false 2023-08-10T06:30:54,350 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.HRegion(2520): Flush status journal for 41ac6ed181c2526bbcdb3ef2c04e3054: 2023-08-10T06:30:54,350 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. for snapshot_1691649053720_default_test-1691649035399 completed. 2023-08-10T06:30:54,350 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(240): Storing 'test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054.' region-info for snapshot=snapshot_1691649053720_default_test-1691649035399 2023-08-10T06:30:54,350 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:30:54,350 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/f/fa1d8c8d9a7f448881583cf4837e9f5e] hfiles 2023-08-10T06:30:54,350 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/f/fa1d8c8d9a7f448881583cf4837e9f5e for snapshot=snapshot_1691649053720_default_test-1691649035399 2023-08-10T06:30:54,350 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/f3/469af5dd68d44838a76a85d58d415d3a] hfiles 2023-08-10T06:30:54,350 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/f3/469af5dd68d44838a76a85d58d415d3a for snapshot=snapshot_1691649053720_default_test-1691649035399 2023-08-10T06:30:54,759 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:30:54,759 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=40}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=40 2023-08-10T06:30:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster(3994): Remote procedure done, pid=40 2023-08-10T06:30:54,760 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691649053720_default_test-1691649035399 on region 41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:30:54,761 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE; SnapshotRegionProcedure 41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:30:54,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=40, resume processing ppid=39 2023-08-10T06:30:54,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=40, ppid=39, state=SUCCESS; SnapshotRegionProcedure 41ac6ed181c2526bbcdb3ef2c04e3054 in 611 msec 2023-08-10T06:30:54,764 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1691649053720_default_test-1691649035399 table=test-1691649035399 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:30:54,765 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1691649053720_default_test-1691649035399 table=test-1691649035399 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:30:54,765 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1691649053720_default_test-1691649035399 table=test-1691649035399 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:30:54,765 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691649053720_default_test-1691649035399 2023-08-10T06:30:54,766 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/snapshot_1691649053720_default_test-1691649035399 2023-08-10T06:30:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-08-10T06:30:55,185 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1691649053720_default_test-1691649035399 table=test-1691649035399 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:30:55,193 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1691649053720_default_test-1691649035399 table=test-1691649035399 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:30:55,194 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/snapshot_1691649053720_default_test-1691649035399 to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/snapshot_1691649053720_default_test-1691649035399 2023-08-10T06:30:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-08-10T06:30:56,039 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=39, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1691649053720_default_test-1691649035399 table=test-1691649035399 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:30:56,039 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691649053720_default_test-1691649035399 table=test-1691649035399 type=FLUSH ttl=0 }, snapshot procedure id = 39 2023-08-10T06:30:56,041 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=39, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=39, snapshot={ ss=snapshot_1691649053720_default_test-1691649035399 table=test-1691649035399 type=FLUSH ttl=0 } in 2.3060 sec 2023-08-10T06:30:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=39 2023-08-10T06:30:57,844 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: default:test-1691649035399, procId: 39 completed 2023-08-10T06:30:57,845 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1691649057845_ns2_test-16916490353991 table=ns2:test-16916490353991 type=FLUSH ttl=0 } 2023-08-10T06:30:57,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649057845 (current time:1691649057845). 2023-08-10T06:30:57,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:30:57,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1691649057845_ns2_test-16916490353991 VERSION not specified, setting to 2 2023-08-10T06:30:57,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x494679e7 to 127.0.0.1:55224 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-08-10T06:30:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3265fb8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:30:57,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:30:57,855 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:50874, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:30:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x494679e7 to 127.0.0.1:55224 2023-08-10T06:30:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:30:57,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:30:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=41, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1691649057845_ns2_test-16916490353991 table=ns2:test-16916490353991 type=FLUSH ttl=0 } 2023-08-10T06:30:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_1691649057845_ns2_test-16916490353991 table=ns2:test-16916490353991 type=FLUSH ttl=0 }, snapshot procedure id = 41 2023-08-10T06:30:57,859 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1691649057845_ns2_test-16916490353991 table=ns2:test-16916490353991 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:30:57,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-08-10T06:30:57,860 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1691649057845_ns2_test-16916490353991 table=ns2:test-16916490353991 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:30:57,862 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1691649057845_ns2_test-16916490353991 table=ns2:test-16916490353991 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:30:57,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-08-10T06:30:58,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-08-10T06:30:58,271 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1691649057845_ns2_test-16916490353991 table=ns2:test-16916490353991 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:30:58,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE; SnapshotRegionProcedure ab0a54958c666e251775ab33a9efc343}] 2023-08-10T06:30:58,273 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=42, ppid=41, state=RUNNABLE; SnapshotRegionProcedure ab0a54958c666e251775ab33a9efc343 2023-08-10T06:30:58,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40209 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=42 2023-08-10T06:30:58,425 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343. 2023-08-10T06:30:58,426 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HRegion(2819): Flushing ab0a54958c666e251775ab33a9efc343 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-08-10T06:30:58,442 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns2/test-16916490353991/ab0a54958c666e251775ab33a9efc343/.tmp/f/9df456b3ebab4661981987c1a670948b is 37, key is row10/f:q1/1691649040256/Put/seqid=0 2023-08-10T06:30:58,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-08-10T06:30:58,852 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns2/test-16916490353991/ab0a54958c666e251775ab33a9efc343/.tmp/f/9df456b3ebab4661981987c1a670948b 2023-08-10T06:30:58,859 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9df456b3ebab4661981987c1a670948b 2023-08-10T06:30:58,860 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns2/test-16916490353991/ab0a54958c666e251775ab33a9efc343/.tmp/f/9df456b3ebab4661981987c1a670948b as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns2/test-16916490353991/ab0a54958c666e251775ab33a9efc343/f/9df456b3ebab4661981987c1a670948b 2023-08-10T06:30:58,867 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns2/test-16916490353991/ab0a54958c666e251775ab33a9efc343/f/9df456b3ebab4661981987c1a670948b, entries=99, sequenceid=103, filesize=8.2 K 2023-08-10T06:30:58,868 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for ab0a54958c666e251775ab33a9efc343 in 442ms, sequenceid=103, compaction requested=false 2023-08-10T06:30:58,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.HRegion(2520): Flush status journal for ab0a54958c666e251775ab33a9efc343: 2023-08-10T06:30:58,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343. for snapshot_1691649057845_ns2_test-16916490353991 completed. 2023-08-10T06:30:58,870 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343.' region-info for snapshot=snapshot_1691649057845_ns2_test-16916490353991 2023-08-10T06:30:58,870 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:30:58,870 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns2/test-16916490353991/ab0a54958c666e251775ab33a9efc343/f/9df456b3ebab4661981987c1a670948b] hfiles 2023-08-10T06:30:58,870 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns2/test-16916490353991/ab0a54958c666e251775ab33a9efc343/f/9df456b3ebab4661981987c1a670948b for snapshot=snapshot_1691649057845_ns2_test-16916490353991 2023-08-10T06:30:58,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-08-10T06:30:59,276 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343. 2023-08-10T06:30:59,276 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=42}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=42 2023-08-10T06:30:59,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster(3994): Remote procedure done, pid=42 2023-08-10T06:30:59,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1691649057845_ns2_test-16916490353991 on region ab0a54958c666e251775ab33a9efc343 2023-08-10T06:30:59,278 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=42, ppid=41, state=RUNNABLE; SnapshotRegionProcedure ab0a54958c666e251775ab33a9efc343 2023-08-10T06:30:59,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=42, resume processing ppid=41 2023-08-10T06:30:59,280 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1691649057845_ns2_test-16916490353991 table=ns2:test-16916490353991 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:30:59,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=42, ppid=41, state=SUCCESS; SnapshotRegionProcedure ab0a54958c666e251775ab33a9efc343 in 1.0080 sec 2023-08-10T06:30:59,281 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1691649057845_ns2_test-16916490353991 table=ns2:test-16916490353991 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:30:59,282 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1691649057845_ns2_test-16916490353991 table=ns2:test-16916490353991 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:30:59,282 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1691649057845_ns2_test-16916490353991 2023-08-10T06:30:59,283 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/snapshot_1691649057845_ns2_test-16916490353991 2023-08-10T06:30:59,694 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1691649057845_ns2_test-16916490353991 table=ns2:test-16916490353991 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:30:59,701 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1691649057845_ns2_test-16916490353991 table=ns2:test-16916490353991 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:30:59,702 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/snapshot_1691649057845_ns2_test-16916490353991 to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/snapshot_1691649057845_ns2_test-16916490353991 2023-08-10T06:30:59,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-08-10T06:31:00,524 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1691649057845_ns2_test-16916490353991 table=ns2:test-16916490353991 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:31:00,524 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_1691649057845_ns2_test-16916490353991 table=ns2:test-16916490353991 type=FLUSH ttl=0 }, snapshot procedure id = 41 2023-08-10T06:31:00,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=41, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=snapshot_1691649057845_ns2_test-16916490353991 table=ns2:test-16916490353991 type=FLUSH ttl=0 } in 2.6670 sec 2023-08-10T06:31:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=41 2023-08-10T06:31:01,968 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] client.HBaseAdmin$TableFuture(3590): Operation: SNAPSHOT, Table Name: ns2:test-16916490353991, procId: 41 completed 2023-08-10T06:31:01,968 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1691649045860 2023-08-10T06:31:01,968 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-08-10T06:31:01,970 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-08-10T06:31:01,972 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691649045860_ns2:test-16916490353991 2023-08-10T06:31:01,973 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691649057845_ns2_test-16916490353991 to hdfs://localhost:43579/backupUT/backup_1691649045860/ns2/test-16916490353991/ 2023-08-10T06:31:01,982 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:31:02,012 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:43579, inputRoot=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a 2023-08-10T06:31:02,012 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,46449,1691649023646_-1868607609_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:43579/backupUT/backup_1691649045860/ns2/test-16916490353991, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:43579/backupUT/backup_1691649045860/ns2/test-16916490353991/.hbase-snapshot/.tmp/snapshot_1691649057845_ns2_test-16916490353991 2023-08-10T06:31:02,012 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:43579, inputRoot=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a, snapshotDir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/snapshot_1691649057845_ns2_test-16916490353991. 2023-08-10T06:31:02,023 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/snapshot_1691649057845_ns2_test-16916490353991 to hdfs://localhost:43579/backupUT/backup_1691649045860/ns2/test-16916490353991/.hbase-snapshot/.tmp/snapshot_1691649057845_ns2_test-16916490353991 2023-08-10T06:31:02,450 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:02,451 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:02,452 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:02,452 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:03,251 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-08-10T06:31:03,254 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43866, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-08-10T06:31:03,453 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/hadoop-7974265227608840821.jar 2023-08-10T06:31:03,454 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:03,454 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:03,455 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:03,456 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:03,456 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:03,457 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:03,457 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:03,458 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:03,458 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:03,460 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:31:03,461 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:31:03,461 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:31:03,462 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:31:03,463 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:31:03,464 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:31:03,464 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:31:03,464 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:31:03,465 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:31:03,465 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:31:03,466 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:31:03,466 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:31:03,471 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:03,471 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:03,472 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:03,472 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:03,472 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:03,473 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:03,474 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:03,533 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:31:03,555 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691649057845_ns2_test-16916490353991' hfile list 2023-08-10T06:31:03,565 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-08-10T06:31:03,768 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/mapreduce.cluster.local.dir/1691649063722/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:31:03,768 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/mapreduce.cluster.local.dir/1691649063722/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:31:03,937 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-08-10T06:31:03,966 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns2/test-16916490353991/ab0a54958c666e251775ab33a9efc343/f/9df456b3ebab4661981987c1a670948b output=hdfs://localhost:43579/backupUT/backup_1691649045860/ns2/test-16916490353991/archive/data/ns2/test-16916490353991/ab0a54958c666e251775ab33a9efc343/f/9df456b3ebab4661981987c1a670948b 2023-08-10T06:31:03,967 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-08-10T06:31:04,847 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:31:04,848 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:31:04,857 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691649057845_ns2_test-16916490353991 2023-08-10T06:31:04,857 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691649057845_ns2_test-16916490353991 finished. 2023-08-10T06:31:04,857 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1691649045860_test-1691649035399 2023-08-10T06:31:04,857 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1691649053720_default_test-1691649035399 to hdfs://localhost:43579/backupUT/backup_1691649045860/default/test-1691649035399/ 2023-08-10T06:31:04,857 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-08-10T06:31:04,880 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:43579, inputRoot=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a 2023-08-10T06:31:04,880 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,46449,1691649023646_-186056953_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:43579/backupUT/backup_1691649045860/default/test-1691649035399, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:43579/backupUT/backup_1691649045860/default/test-1691649035399/.hbase-snapshot/.tmp/snapshot_1691649053720_default_test-1691649035399 2023-08-10T06:31:04,880 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:43579, inputRoot=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a, snapshotDir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/snapshot_1691649053720_default_test-1691649035399. 2023-08-10T06:31:04,892 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/snapshot_1691649053720_default_test-1691649035399 to hdfs://localhost:43579/backupUT/backup_1691649045860/default/test-1691649035399/.hbase-snapshot/.tmp/snapshot_1691649053720_default_test-1691649035399 2023-08-10T06:31:05,313 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:05,314 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:05,314 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:05,315 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:06,301 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/hadoop-1363606087477083938.jar 2023-08-10T06:31:06,302 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:06,302 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:06,302 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:06,303 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:06,303 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:06,304 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:06,304 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:06,304 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:06,305 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:06,306 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:31:06,306 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:31:06,307 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:31:06,307 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:31:06,308 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:31:06,308 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:31:06,308 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:31:06,309 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:31:06,309 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:31:06,310 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:31:06,310 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:31:06,311 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:31:06,311 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:06,312 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:06,312 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:06,313 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:06,313 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-08-10T06:31:06,314 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:06,315 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:06,341 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-08-10T06:31:06,358 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1691649053720_default_test-1691649035399' hfile list 2023-08-10T06:31:06,361 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] snapshot.ExportSnapshot(674): export split=0 size=13.4 K 2023-08-10T06:31:06,432 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/mapreduce.cluster.local.dir/1691649066398/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:31:06,433 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/mapreduce.cluster.local.dir/1691649066398/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:31:06,492 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-08-10T06:31:06,543 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/f/fa1d8c8d9a7f448881583cf4837e9f5e output=hdfs://localhost:43579/backupUT/backup_1691649045860/default/test-1691649035399/archive/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/f/fa1d8c8d9a7f448881583cf4837e9f5e 2023-08-10T06:31:06,543 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-08-10T06:31:06,976 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/f3/469af5dd68d44838a76a85d58d415d3a output=hdfs://localhost:43579/backupUT/backup_1691649045860/default/test-1691649035399/archive/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/f3/469af5dd68d44838a76a85d58d415d3a 2023-08-10T06:31:06,977 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=5370 (5.2 K) time=0sec 5.121M/sec 2023-08-10T06:31:07,471 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-08-10T06:31:07,473 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-08-10T06:31:07,482 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1691649053720_default_test-1691649035399 2023-08-10T06:31:07,482 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1691649053720_default_test-1691649035399 finished. 2023-08-10T06:31:07,482 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1014): ns2:test-16916490353991 2023-08-10T06:31:07,483 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1014): test-1691649035399 2023-08-10T06:31:07,556 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649045860 2023-08-10T06:31:07,556 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:31:07,969 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:43579/backupUT/backup_1691649045860/.backup.manifest 2023-08-10T06:31:07,969 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1691649045860 2023-08-10T06:31:07,969 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-08-10T06:31:08,375 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:43579/backupUT/backup_1691649045860/.backup.manifest 2023-08-10T06:31:08,375 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(358): Backup backup_1691649045860 finished: type=FULL,tablelist=ns2:test-16916490353991;test-1691649035399,targetRootDir=hdfs://localhost:43579/backupUT,startts=1691649053446,completets=1691649067554,bytescopied=0 2023-08-10T06:31:08,375 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-08-10T06:31:08,375 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691649057845_ns2_test-16916490353991 2023-08-10T06:31:08,385 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691649057845_ns2_test-16916490353991" 2023-08-10T06:31:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691649057845_ns2_test-16916490353991 2023-08-10T06:31:08,391 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691649057845_ns2_test-16916490353991 for backup backup_1691649045860 succeeded. 2023-08-10T06:31:08,391 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1691649053720_default_test-1691649035399 2023-08-10T06:31:08,392 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_1691649053720_default_test-1691649035399" 2023-08-10T06:31:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_1691649053720_default_test-1691649035399 2023-08-10T06:31:08,398 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1691649053720_default_test-1691649035399 for backup backup_1691649045860 succeeded. 2023-08-10T06:31:08,399 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:31:08,411 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:31:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:31:08,414 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:31:08,421 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:31:08,438 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(376): Backup backup_1691649045860 completed. 2023-08-10T06:31:08,559 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] backup.TestIncrementalBackupWithFailures(104): writing 99 rows to test-1691649035399 2023-08-10T06:31:08,574 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] backup.TestIncrementalBackupWithFailures(108): written 99 rows to test-1691649035399 2023-08-10T06:31:08,589 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] backup.TestIncrementalBackupWithFailures(120): written 5 rows to ns2:test-16916490353991 2023-08-10T06:31:08,590 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 0 2023-08-10T06:31:08,629 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47953c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:08,630 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:31:08,631 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43880, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:31:08,635 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:08,636 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52350, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:08,700 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:31:08,703 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_0 in testing java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:123) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-08-10T06:31:08,704 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(240): BackupId=backup_1691649068665,startts=0,failedts=1691649068704,failedphase=REQUEST,failedmessage=Failed stage stage_0 in testing 2023-08-10T06:31:08,704 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-08-10T06:31:08,706 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1530): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2023-08-10T06:31:08,706 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:31:08,708 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1559): Snapshot snapshot_backup_system does not exists 2023-08-10T06:31:08,708 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-08-10T06:31:08,709 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:31:08,711 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(246): Backup backup_1691649068665 failed. Backup session finished. Status: FAILURE 2023-08-10T06:31:08,711 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:08,712 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:123) ~[test-classes/:?] ... 41 more 2023-08-10T06:31:08,716 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 1 2023-08-10T06:31:08,734 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ca4b2e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:08,735 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:31:08,736 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43894, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:31:08,739 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:08,741 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52362, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:08,789 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:31:08,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:08,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649068792 (current time:1691649068792). 2023-08-10T06:31:08,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:31:08,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:31:08,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ae297d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:08,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:08,802 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:52376, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:08,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:31:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=43, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 43 2023-08-10T06:31:08,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-08-10T06:31:08,805 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:31:08,806 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:31:08,808 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:31:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-08-10T06:31:09,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-08-10T06:31:09,215 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:31:09,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 4cb36c501bd471befb9ec81ac2da937f}] 2023-08-10T06:31:09,217 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:09,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40209 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=44 2023-08-10T06:31:09,368 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:09,369 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(2819): Flushing 4cb36c501bd471befb9ec81ac2da937f 2/2 column families, dataSize=1.83 KB heapSize=3.35 KB 2023-08-10T06:31:09,387 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/meta/6cbd9d64f8534727b32d2198cce57056 is 171, key is trslm:hdfs://localhost:43579/backupUT\x00ns2:test-16916490353991/meta:log-roll-map/1691649067542/Put/seqid=0 2023-08-10T06:31:09,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-08-10T06:31:09,792 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=785 B at sequenceid=20 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/meta/6cbd9d64f8534727b32d2198cce57056 2023-08-10T06:31:09,799 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6cbd9d64f8534727b32d2198cce57056 2023-08-10T06:31:09,801 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/f6e3a0a30de744a4a3d21c98964e6218 is 459, key is session:backup_1691649045860/session:context/1691649068420/Put/seqid=0 2023-08-10T06:31:09,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-08-10T06:31:10,207 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.07 KB at sequenceid=20 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/f6e3a0a30de744a4a3d21c98964e6218 2023-08-10T06:31:10,213 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f6e3a0a30de744a4a3d21c98964e6218 2023-08-10T06:31:10,215 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/meta/6cbd9d64f8534727b32d2198cce57056 as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056 2023-08-10T06:31:10,221 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056, entries=6, sequenceid=20, filesize=5.8 K 2023-08-10T06:31:10,222 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/f6e3a0a30de744a4a3d21c98964e6218 as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f6e3a0a30de744a4a3d21c98964e6218 2023-08-10T06:31:10,227 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f6e3a0a30de744a4a3d21c98964e6218, entries=3, sequenceid=20, filesize=5.7 K 2023-08-10T06:31:10,228 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(3022): Finished flush of dataSize ~1.83 KB/1879, heapSize ~3.32 KB/3400, currentSize=0 B/0 for 4cb36c501bd471befb9ec81ac2da937f in 859ms, sequenceid=20, compaction requested=false 2023-08-10T06:31:10,228 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.HRegion(2520): Flush status journal for 4cb36c501bd471befb9ec81ac2da937f: 2023-08-10T06:31:10,228 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. for snapshot_backup_system completed. 2023-08-10T06:31:10,229 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:31:10,229 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:31:10,229 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056] hfiles 2023-08-10T06:31:10,229 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056 for snapshot=snapshot_backup_system 2023-08-10T06:31:10,229 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/34d64473585d4371aa81253c6048c0d8, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f6e3a0a30de744a4a3d21c98964e6218] hfiles 2023-08-10T06:31:10,229 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/34d64473585d4371aa81253c6048c0d8 for snapshot=snapshot_backup_system 2023-08-10T06:31:10,229 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f6e3a0a30de744a4a3d21c98964e6218 for snapshot=snapshot_backup_system 2023-08-10T06:31:10,634 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:10,635 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=44}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=44 2023-08-10T06:31:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster(3994): Remote procedure done, pid=44 2023-08-10T06:31:10,635 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:10,637 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE; SnapshotRegionProcedure 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:10,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=44, resume processing ppid=43 2023-08-10T06:31:10,639 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:31:10,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=44, ppid=43, state=SUCCESS; SnapshotRegionProcedure 4cb36c501bd471befb9ec81ac2da937f in 1.4220 sec 2023-08-10T06:31:10,639 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:31:10,640 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:31:10,640 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:31:10,641 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:31:10,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-08-10T06:31:10,986 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] procedure2.ProcedureExecutor(1033): Stored pid=45, state=RUNNABLE:MODIFY_TABLE_DESCRIPTOR_PREPARE; InitializeStoreFileTrackerProcedure table=test-1691649035399 2023-08-10T06:31:11,053 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:31:11,060 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:31:11,061 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:31:11,401 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/.tabledesc/.tableinfo.0000000002.598 2023-08-10T06:31:11,401 INFO [PEWorker-2 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/.tabledesc/.tableinfo.0000000003.640 2023-08-10T06:31:11,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=45, state=SUCCESS; InitializeStoreFileTrackerProcedure table=test-1691649035399 in 420 msec 2023-08-10T06:31:11,880 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=43, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:31:11,880 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 43 2023-08-10T06:31:11,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=43, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=43, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 3.0770 sec 2023-08-10T06:31:12,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=43 2023-08-10T06:31:12,912 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(115): Backup backup_1691649068765 started at 1691649072912. 2023-08-10T06:31:12,916 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(119): Backup session backup_1691649068765 has been started. 2023-08-10T06:31:12,916 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_1 in testing java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:126) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-08-10T06:31:12,916 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(240): BackupId=backup_1691649068765,startts=1691649072912,failedts=1691649072916,failedphase=REQUEST,failedmessage=Failed stage stage_1 in testing 2023-08-10T06:31:12,917 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-08-10T06:31:12,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable backup:system 2023-08-10T06:31:12,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=46, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-08-10T06:31:12,939 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649072939"}]},"ts":"1691649072939"} 2023-08-10T06:31:12,940 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-08-10T06:31:12,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-08-10T06:31:12,942 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-08-10T06:31:12,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, UNASSIGN}] 2023-08-10T06:31:12,945 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=47, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, UNASSIGN 2023-08-10T06:31:12,946 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=47 updating hbase:meta row=4cb36c501bd471befb9ec81ac2da937f, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:12,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE; CloseRegionProcedure 4cb36c501bd471befb9ec81ac2da937f, server=jenkins-hbase3.apache.org,40209,1691649026642}] 2023-08-10T06:31:13,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-08-10T06:31:13,099 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] handler.UnassignRegionHandler(119): Close 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:13,099 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1668): Closing 4cb36c501bd471befb9ec81ac2da937f, disabling compactions & flushes 2023-08-10T06:31:13,099 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1690): Closing region backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:13,099 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:13,099 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. after waiting 0 ms 2023-08-10T06:31:13,099 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:13,099 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(2819): Flushing 4cb36c501bd471befb9ec81ac2da937f 2/2 column families, dataSize=214 B heapSize=800 B 2023-08-10T06:31:13,103 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/6d0857f895724d4ea5de7a4e191d571d is 218, key is session:backup_1691649068765/session:context/1691649072913/Put/seqid=0 2023-08-10T06:31:13,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-08-10T06:31:13,509 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/6d0857f895724d4ea5de7a4e191d571d 2023-08-10T06:31:13,515 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6d0857f895724d4ea5de7a4e191d571d 2023-08-10T06:31:13,516 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/6d0857f895724d4ea5de7a4e191d571d as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/6d0857f895724d4ea5de7a4e191d571d 2023-08-10T06:31:13,522 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/6d0857f895724d4ea5de7a4e191d571d, entries=1, sequenceid=24, filesize=5.2 K 2023-08-10T06:31:13,523 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(3022): Finished flush of dataSize ~214 B/214, heapSize ~528 B/528, currentSize=0 B/0 for 4cb36c501bd471befb9ec81ac2da937f in 424ms, sequenceid=24, compaction requested=true 2023-08-10T06:31:13,525 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6cbd9d64f8534727b32d2198cce57056 2023-08-10T06:31:13,527 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 34d64473585d4371aa81253c6048c0d8 2023-08-10T06:31:13,529 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f6e3a0a30de744a4a3d21c98964e6218 2023-08-10T06:31:13,530 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6d0857f895724d4ea5de7a4e191d571d 2023-08-10T06:31:13,535 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/recovered.edits/27.seqid, newMaxSeqId=27, maxSeqId=1 2023-08-10T06:31:13,535 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:31:13,536 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1909): Closed backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:13,536 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1622): Region close journal for 4cb36c501bd471befb9ec81ac2da937f: 2023-08-10T06:31:13,538 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] handler.UnassignRegionHandler(163): Closed 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:13,539 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=47 updating hbase:meta row=4cb36c501bd471befb9ec81ac2da937f, regionState=CLOSED 2023-08-10T06:31:13,542 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=48, resume processing ppid=47 2023-08-10T06:31:13,542 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=48, ppid=47, state=SUCCESS; CloseRegionProcedure 4cb36c501bd471befb9ec81ac2da937f, server=jenkins-hbase3.apache.org,40209,1691649026642 in 593 msec 2023-08-10T06:31:13,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=47, resume processing ppid=46 2023-08-10T06:31:13,543 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=47, ppid=46, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, UNASSIGN in 599 msec 2023-08-10T06:31:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-08-10T06:31:13,544 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649073544"}]},"ts":"1691649073544"} 2023-08-10T06:31:13,545 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-08-10T06:31:13,547 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-08-10T06:31:13,549 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=46, state=SUCCESS; DisableTableProcedure table=backup:system in 618 msec 2023-08-10T06:31:13,812 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-08-10T06:31:14,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=46 2023-08-10T06:31:14,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1691649074053 table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649074054 (current time:1691649074054). 2023-08-10T06:31:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:31:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1691649074053 VERSION not specified, setting to 2 2023-08-10T06:31:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54e0f9b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:14,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:14,062 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36232, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:14,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:31:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=49, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649074053 table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:14,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649074053 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 49 2023-08-10T06:31:14,066 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649074053 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:31:14,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-08-10T06:31:14,067 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649074053 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:31:14,069 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649074053 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:31:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-08-10T06:31:14,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-08-10T06:31:14,475 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649074053 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-08-10T06:31:14,477 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-08-10T06:31:14,477 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-08-10T06:31:14,477 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:31:14,478 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056] hfiles 2023-08-10T06:31:14,478 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056 2023-08-10T06:31:14,478 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:31:14,480 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/34d64473585d4371aa81253c6048c0d8, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/6d0857f895724d4ea5de7a4e191d571d, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f6e3a0a30de744a4a3d21c98964e6218] hfiles 2023-08-10T06:31:14,480 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/34d64473585d4371aa81253c6048c0d8 2023-08-10T06:31:14,480 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/6d0857f895724d4ea5de7a4e191d571d 2023-08-10T06:31:14,480 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f6e3a0a30de744a4a3d21c98964e6218 2023-08-10T06:31:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-08-10T06:31:14,886 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 4cb36c501bd471befb9ec81ac2da937f, NAME => 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-08-10T06:31:14,887 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649074053 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:31:14,888 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649074053 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:31:14,888 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1691649074053 2023-08-10T06:31:14,889 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1691649074053 2023-08-10T06:31:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-08-10T06:31:15,300 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649074053 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:31:15,308 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649074053 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:31:15,309 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1691649074053 to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1691649074053 2023-08-10T06:31:16,128 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649074053 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:31:16,128 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649074053 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 49 2023-08-10T06:31:16,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=49, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649074053 table=backup:system type=FLUSH ttl=0 } in 2.0650 sec 2023-08-10T06:31:16,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=49 2023-08-10T06:31:16,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=50, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691649068792 type: FLUSH version: 2 ttl: 0 ) 2023-08-10T06:31:16,199 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-08-10T06:31:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:31:16,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:31:16,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:31:16,613 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2023-08-10T06:31:16,613 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-08-10T06:31:16,615 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:16,620 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691649068792 type: FLUSH version: 2 ttl: 0 2023-08-10T06:31:16,621 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system 2023-08-10T06:31:16,624 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-08-10T06:31:16,624 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:16,629 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:31:16,638 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/6d0857f895724d4ea5de7a4e191d571d to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/archive/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/6d0857f895724d4ea5de7a4e191d571d 2023-08-10T06:31:16,639 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:31:16,639 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691649068792 type: FLUSH version: 2 ttl: 0 2023-08-10T06:31:16,640 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691649076640"}]},"ts":"9223372036854775807"} 2023-08-10T06:31:16,643 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:31:16,643 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 4cb36c501bd471befb9ec81ac2da937f, NAME => 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:31:16,643 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1691649076641"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649076641"}]},"ts":"1691649076641"} 2023-08-10T06:31:16,644 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:31:16,644 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-08-10T06:31:16,644 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 4cb36c501bd471befb9ec81ac2da937f, NAME => 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:31:16,645 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-08-10T06:31:16,647 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=50, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691649068792 type: FLUSH version: 2 ttl: 0 ) in 457 msec 2023-08-10T06:31:16,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=50 2023-08-10T06:31:16,807 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1691649074053" 2023-08-10T06:31:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1691649074053 2023-08-10T06:31:16,817 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster$11(2630): Client=jenkins//172.31.12.81 enable backup:system 2023-08-10T06:31:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=51, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-08-10T06:31:16,827 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-08-10T06:31:16,828 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649076827"}]},"ts":"1691649076827"} 2023-08-10T06:31:16,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=51 2023-08-10T06:31:16,829 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-08-10T06:31:16,837 DEBUG [PEWorker-2 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-08-10T06:31:16,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, ASSIGN}] 2023-08-10T06:31:16,839 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, ASSIGN 2023-08-10T06:31:16,840 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-08-10T06:31:16,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=51 2023-08-10T06:31:16,991 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=4cb36c501bd471befb9ec81ac2da937f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:16,993 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; OpenRegionProcedure 4cb36c501bd471befb9ec81ac2da937f, server=jenkins-hbase3.apache.org,40209,1691649026642}] 2023-08-10T06:31:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=51 2023-08-10T06:31:17,148 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(130): Open backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:17,148 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7254): Opening region: {ENCODED => 4cb36c501bd471befb9ec81ac2da937f, NAME => 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:31:17,149 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:31:17,149 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:17,149 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(888): Instantiated backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:31:17,149 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7296): checking encryption for 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:17,149 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7299): checking classloading for 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:17,151 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:17,152 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4cb36c501bd471befb9ec81ac2da937f columnFamilyName meta 2023-08-10T06:31:17,152 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:31:17,159 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056 2023-08-10T06:31:17,159 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.HStore(324): Store=4cb36c501bd471befb9ec81ac2da937f/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:31:17,159 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:17,160 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4cb36c501bd471befb9ec81ac2da937f columnFamilyName session 2023-08-10T06:31:17,160 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:31:17,166 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/34d64473585d4371aa81253c6048c0d8 2023-08-10T06:31:17,172 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f6e3a0a30de744a4a3d21c98964e6218 2023-08-10T06:31:17,172 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.HStore(324): Store=4cb36c501bd471befb9ec81ac2da937f/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:31:17,173 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:17,174 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:17,176 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:31:17,177 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1076): writing seq id for 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:17,178 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): Opened 4cb36c501bd471befb9ec81ac2da937f; next sequenceid=28; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10157166080, jitterRate=-0.05404019355773926}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:31:17,179 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(992): Region open journal for 4cb36c501bd471befb9ec81ac2da937f: 2023-08-10T06:31:17,179 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f., pid=53, masterSystemTime=1691649077145 2023-08-10T06:31:17,181 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:17,181 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(158): Opened backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:17,182 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=4cb36c501bd471befb9ec81ac2da937f, regionState=OPEN, openSeqNum=28, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:17,184 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=53, resume processing ppid=52 2023-08-10T06:31:17,184 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=53, ppid=52, state=SUCCESS; OpenRegionProcedure 4cb36c501bd471befb9ec81ac2da937f, server=jenkins-hbase3.apache.org,40209,1691649026642 in 190 msec 2023-08-10T06:31:17,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=52, resume processing ppid=51 2023-08-10T06:31:17,186 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=52, ppid=51, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, ASSIGN in 347 msec 2023-08-10T06:31:17,187 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649077187"}]},"ts":"1691649077187"} 2023-08-10T06:31:17,188 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-08-10T06:31:17,189 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-08-10T06:31:17,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=51, state=SUCCESS; EnableTableProcedure table=backup:system in 371 msec 2023-08-10T06:31:17,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=51 2023-08-10T06:31:17,432 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-08-10T06:31:17,432 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:31:17,438 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:31:17,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:31:17,441 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:31:17,441 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-08-10T06:31:17,443 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:31:17,450 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(246): Backup backup_1691649068765 failed. Backup session finished. Status: FAILURE 2023-08-10T06:31:17,450 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:17,450 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:126) ~[test-classes/:?] ... 41 more 2023-08-10T06:31:17,457 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 2 2023-08-10T06:31:17,484 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ec79f36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:17,485 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:31:17,487 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:38214, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:31:17,491 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:17,492 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36246, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:17,558 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:31:17,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:17,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649077562 (current time:1691649077562). 2023-08-10T06:31:17,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:31:17,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:31:17,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e931311, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:17,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:17,574 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:36248, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:31:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=54, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 54 2023-08-10T06:31:17,578 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:31:17,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:31:17,579 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:31:17,580 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:31:17,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:31:17,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:31:17,990 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:31:17,990 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure 4cb36c501bd471befb9ec81ac2da937f}] 2023-08-10T06:31:17,992 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:18,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40209 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=55 2023-08-10T06:31:18,143 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:18,145 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2819): Flushing 4cb36c501bd471befb9ec81ac2da937f 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-08-10T06:31:18,159 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/f15ed284b4d44c27b14fc474f7eaa879 is 256, key is session:backup_1691649068765/session:context/1691649077442/Put/seqid=0 2023-08-10T06:31:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:31:18,564 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/f15ed284b4d44c27b14fc474f7eaa879 2023-08-10T06:31:18,570 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f15ed284b4d44c27b14fc474f7eaa879 2023-08-10T06:31:18,571 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/f15ed284b4d44c27b14fc474f7eaa879 as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f15ed284b4d44c27b14fc474f7eaa879 2023-08-10T06:31:18,578 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f15ed284b4d44c27b14fc474f7eaa879, entries=2, sequenceid=33, filesize=5.2 K 2023-08-10T06:31:18,579 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for 4cb36c501bd471befb9ec81ac2da937f in 434ms, sequenceid=33, compaction requested=true 2023-08-10T06:31:18,579 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-08-10T06:31:18,580 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2520): Flush status journal for 4cb36c501bd471befb9ec81ac2da937f: 2023-08-10T06:31:18,580 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. for snapshot_backup_system completed. 2023-08-10T06:31:18,580 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:31:18,580 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:31:18,580 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056] hfiles 2023-08-10T06:31:18,580 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056 for snapshot=snapshot_backup_system 2023-08-10T06:31:18,580 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/34d64473585d4371aa81253c6048c0d8, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f6e3a0a30de744a4a3d21c98964e6218, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f15ed284b4d44c27b14fc474f7eaa879] hfiles 2023-08-10T06:31:18,580 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/34d64473585d4371aa81253c6048c0d8 for snapshot=snapshot_backup_system 2023-08-10T06:31:18,581 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f6e3a0a30de744a4a3d21c98964e6218 for snapshot=snapshot_backup_system 2023-08-10T06:31:18,581 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f15ed284b4d44c27b14fc474f7eaa879 for snapshot=snapshot_backup_system 2023-08-10T06:31:18,591 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:18,591 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=55 2023-08-10T06:31:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster(3994): Remote procedure done, pid=55 2023-08-10T06:31:18,592 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:18,593 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:18,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=55, resume processing ppid=54 2023-08-10T06:31:18,595 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:31:18,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=55, ppid=54, state=SUCCESS; SnapshotRegionProcedure 4cb36c501bd471befb9ec81ac2da937f in 604 msec 2023-08-10T06:31:18,596 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:31:18,597 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:31:18,597 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:31:18,597 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:31:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:31:19,010 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:31:19,017 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:31:19,018 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:31:19,087 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:31:19,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:31:19,835 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:31:19,835 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 54 2023-08-10T06:31:19,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=54, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2600 sec 2023-08-10T06:31:21,551 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-08-10T06:31:21,551 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-08-10T06:31:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=54 2023-08-10T06:31:21,685 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(115): Backup backup_1691649077525 started at 1691649081685. 2023-08-10T06:31:21,687 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(119): Backup session backup_1691649077525 has been started. 2023-08-10T06:31:21,688 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] backup.TestBackupBase$IncrementalTableBackupClientForTest(128): For incremental backup, current table set is [ns2:test-16916490353991, test-1691649035399] 2023-08-10T06:31:21,691 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(76): StartCode 1691649030774for backupID backup_1691649077525 2023-08-10T06:31:21,691 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-08-10T06:31:21,692 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:31:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:31:21,692 INFO [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:31:21,692 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:31:21,693 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:31:21,693 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:31:21,694 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:31:21,695 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:21,695 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:31:21,695 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:31:21,696 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:31:21,696 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:31:21,696 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:31:21,696 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:31:21,696 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:31:21,697 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:31:21,697 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:31:21,697 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:31:21,697 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:31:21,698 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:31:21,698 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:31:21,698 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,40209,1691649026642' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:31:21,700 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:21,700 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:21,700 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:31:21,700 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:31:21,700 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:31:21,700 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:31:21,701 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:31:21,701 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:21,701 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:21,702 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:31:21,702 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,40209,1691649026642' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:31:21,702 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@5106a8f0[Count = 0] remaining members to acquire global barrier 2023-08-10T06:31:21,702 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:31:21,702 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:21,703 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:21,703 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:21,703 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:31:21,703 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:31:21,703 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:31:21,704 DEBUG [rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:21,704 INFO [rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691649053543 highest: 1691649053543 on jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:21,704 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1:(num 1691649053506) roll requested 2023-08-10T06:31:21,715 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649081704, exclude list is [], retry=0 2023-08-10T06:31:21,719 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK] 2023-08-10T06:31:21,721 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 with entries=7, filesize=1.16 KB; new WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649081704 2023-08-10T06:31:21,721 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK]] 2023-08-10T06:31:21,721 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 is not closed yet, will try archiving it next time 2023-08-10T06:31:21,721 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta:.meta(num 1691649053525) roll requested 2023-08-10T06:31:21,723 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 not finished, retry = 0 2023-08-10T06:31:21,732 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649081721.meta, exclude list is [], retry=0 2023-08-10T06:31:21,736 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK] 2023-08-10T06:31:21,737 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649053525.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649081721.meta 2023-08-10T06:31:21,737 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK]] 2023-08-10T06:31:21,737 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649053525.meta is not closed yet, will try archiving it next time 2023-08-10T06:31:21,737 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0:(num 1691649053543) roll requested 2023-08-10T06:31:21,739 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649053525.meta not finished, retry = 0 2023-08-10T06:31:21,748 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649081737, exclude list is [], retry=0 2023-08-10T06:31:21,751 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK] 2023-08-10T06:31:21,752 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543 with entries=124, filesize=18.41 KB; new WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649081737 2023-08-10T06:31:21,752 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK]] 2023-08-10T06:31:21,752 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543 is not closed yet, will try archiving it next time 2023-08-10T06:31:21,754 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649028773 to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/oldWALs/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649028773 2023-08-10T06:31:21,764 DEBUG [rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2023-08-10T06:31:21,764 INFO [rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691649081737 on jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:21,772 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:31:21,773 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,40209,1691649026642' in zk 2023-08-10T06:31:21,776 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:31:21,776 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:31:21,776 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:21,776 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:31:21,776 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:31:21,777 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:31:21,777 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:31:21,778 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:31:21,778 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:21,779 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:21,779 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:31:21,779 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:21,780 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:21,781 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,40209,1691649026642': 2023-08-10T06:31:21,781 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,40209,1691649026642' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:31:21,781 INFO [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:31:21,781 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:31:21,781 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:31:21,781 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:31:21,781 INFO [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:31:21,783 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:31:21,783 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:31:21,783 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:31:21,783 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:31:21,783 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:31:21,783 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:31:21,783 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:31:21,783 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:31:21,784 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:21,784 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:31:21,784 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:31:21,784 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:21,785 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:21,785 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:31:21,785 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:21,786 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:21,794 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:31:21,794 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:31:21,794 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:31:21,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:31:21,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:31:21,795 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:31:21,795 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:31:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:31:21,898 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:40209=1691649030774} newestTimestamps: {jenkins-hbase3.apache.org:40209=1691649053543} 2023-08-10T06:31:21,900 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649029122.meta 2023-08-10T06:31:21,900 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649053525.meta 2023-08-10T06:31:21,901 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649081721.meta 2023-08-10T06:31:21,901 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543 2023-08-10T06:31:21,901 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649081737 2023-08-10T06:31:21,901 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 2023-08-10T06:31:21,901 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649081704 2023-08-10T06:31:21,902 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/oldWALs/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649028773 2023-08-10T06:31:22,312 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16916490353991 target: hdfs://localhost:43579/backupUT/backup_1691649077525/ns2/test-16916490353991 descriptor: 'ns2:test-16916490353991', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:31:22,312 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-08-10T06:31:22,314 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16916490353991 2023-08-10T06:31:22,720 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16916490353991 2023-08-10T06:31:23,132 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(146): Attempting to copy table info for:test-1691649035399 target: hdfs://localhost:43579/backupUT/backup_1691649077525/default/test-1691649035399 descriptor: 'test-1691649035399', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:31:23,132 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-08-10T06:31:23,134 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(151): Starting to write region info for table test-1691649035399 2023-08-10T06:31:23,541 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(158): Finished writing region info for table test-1691649035399 2023-08-10T06:31:23,551 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:43579/backupUT/.tmp/backup_1691649077525 from hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543;hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 2023-08-10T06:31:23,559 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25c42b72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:23,561 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:31:23,562 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:33482, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:31:23,577 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-08-10T06:31:23,577 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16916490353991 2023-08-10T06:31:23,581 DEBUG [hconnection-0x59e3ebae-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:23,583 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43226, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:23,586 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16916490353991: ns2:test-16916490353991; 2023-08-10T06:31:23,587 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1691649035399 2023-08-10T06:31:23,589 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1691649035399: test-1691649035399; 2023-08-10T06:31:23,589 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-08-10T06:31:23,590 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/hbase-staging/partitions_76a99cce-13f8-4402-ba81-67a85e740601 2023-08-10T06:31:23,812 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-08-10T06:31:23,812 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-08-10T06:31:24,029 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:24,030 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:24,031 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:24,031 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:25,004 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/hadoop-6871287735093093966.jar 2023-08-10T06:31:25,004 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:25,005 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:25,005 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:25,006 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:25,006 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:25,007 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:25,007 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:25,007 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:25,008 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:25,009 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:31:25,009 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:31:25,010 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:31:25,010 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:31:25,011 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:31:25,011 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:31:25,012 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:31:25,012 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:31:25,013 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:31:25,013 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:31:25,013 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:31:25,014 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:31:25,016 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:25,016 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:25,017 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:25,017 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:25,017 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:25,018 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:25,019 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:25,019 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16916490353991,default:test-1691649035399 2023-08-10T06:31:25,019 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:25,020 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:31:26,004 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/hadoop-8205291085887368740.jar 2023-08-10T06:31:26,025 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543 for WAL files 2023-08-10T06:31:26,026 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543 2023-08-10T06:31:26,027 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 for WAL files 2023-08-10T06:31:26,027 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 2023-08-10T06:31:26,106 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/mapreduce.cluster.local.dir/1691649086072/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:31:26,106 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/mapreduce.cluster.local.dir/1691649086072/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:31:26,185 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543 for hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543 (-9223372036854775808:9223372036854775807) length:18860 2023-08-10T06:31:26,203 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-08-10T06:31:26,203 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-08-10T06:31:26,244 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 for hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 (-9223372036854775808:9223372036854775807) length:1198 2023-08-10T06:31:26,248 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-08-10T06:31:26,248 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-08-10T06:31:26,348 DEBUG [pool-223-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337599b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:26,349 DEBUG [hconnection-0x71eab794-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:26,351 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43228, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:26,353 DEBUG [pool-223-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:26,354 DEBUG [pool-223-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t20] 2023-08-10T06:31:26,355 DEBUG [pool-223-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-08-10T06:31:26,360 DEBUG [pool-223-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/backupUT/.tmp/backup_1691649077525/_temporary/0/_temporary/attempt_local829998135_0003_r_000000_0/ns2/test-16916490353991/f/33cdd629b0aa4caebb2e254d293f783c is 38, key is row-t20/f:q1/1691649068577/Put/seqid=0 2023-08-10T06:31:26,363 WARN [Thread-962 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:40209] Specified favored nodes: [jenkins-hbase3.apache.org:40209] 2023-08-10T06:31:26,794 DEBUG [pool-223-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31196997, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:26,796 DEBUG [hconnection-0x3a5eac60-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:26,797 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43230, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:26,799 DEBUG [pool-223-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:26,800 DEBUG [pool-223-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-test-1691649035399-1-0] 2023-08-10T06:31:26,800 DEBUG [pool-223-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-08-10T06:31:26,816 DEBUG [pool-223-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/backupUT/.tmp/backup_1691649077525/_temporary/0/_temporary/attempt_local829998135_0003_r_000001_0/default/test-1691649035399/f/1a338401db2745dead4b399dd15353bd is 59, key is row-test-1691649035399-1-10/f:q1/1691649068469/Put/seqid=0 2023-08-10T06:31:26,818 WARN [Thread-972 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:40209] Specified favored nodes: [jenkins-hbase3.apache.org:40209] 2023-08-10T06:31:28,142 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:43579/backupUT 2023-08-10T06:31:28,142 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1691649077525 2023-08-10T06:31:28,142 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-08-10T06:31:28,156 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:43579/backupUT/.tmp/backup_1691649077525, hdfs://localhost:43579/backupUT] 2023-08-10T06:31:28,790 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-08-10T06:31:28,792 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1691649077525 2023-08-10T06:31:28,792 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 16003 bytes copied." 2023-08-10T06:31:28,792 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local662922574_0004 completed: true true 2023-08-10T06:31:28,799 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=3790849 FILE: Number of bytes written=5078503 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=194825 HDFS: Number of bytes written=2368946 HDFS: Number of read operations=732 HDFS: Number of large read operations=0 HDFS: Number of write operations=276 Map-Reduce Framework Map input records=10 Map output records=0 Input split bytes=343 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1506803712 File Input Format Counters Bytes Read=2444 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=16003 Bytes Expected=16003 Files Copied=3 DIR_COPY=7 2023-08-10T06:31:28,799 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:43579/backupUT/.tmp/backup_1691649077525 to hdfs://localhost:43579/backupUT finished. 2023-08-10T06:31:28,800 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_2 in testing java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-08-10T06:31:28,801 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(240): BackupId=backup_1691649077525,startts=1691649081685,failedts=1691649088801,failedphase=INCREMENTAL_COPY,failedmessage=Failed stage stage_2 in testing 2023-08-10T06:31:28,801 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-08-10T06:31:28,806 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable backup:system 2023-08-10T06:31:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-08-10T06:31:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-08-10T06:31:28,811 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649088810"}]},"ts":"1691649088810"} 2023-08-10T06:31:28,812 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-08-10T06:31:28,815 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-08-10T06:31:28,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, UNASSIGN}] 2023-08-10T06:31:28,817 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=57, ppid=56, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, UNASSIGN 2023-08-10T06:31:28,818 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=57 updating hbase:meta row=4cb36c501bd471befb9ec81ac2da937f, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:28,819 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE; CloseRegionProcedure 4cb36c501bd471befb9ec81ac2da937f, server=jenkins-hbase3.apache.org,40209,1691649026642}] 2023-08-10T06:31:28,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-08-10T06:31:28,971 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(119): Close 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:28,971 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1668): Closing 4cb36c501bd471befb9ec81ac2da937f, disabling compactions & flushes 2023-08-10T06:31:28,971 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1690): Closing region backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:28,971 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:28,971 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. after waiting 0 ms 2023-08-10T06:31:28,971 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:28,972 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(2819): Flushing 4cb36c501bd471befb9ec81ac2da937f 2/2 column families, dataSize=540 B heapSize=1.24 KB 2023-08-10T06:31:28,987 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/meta/00eae528e68d47598a7cffe1b24c5340 is 116, key is rslogts:hdfs://localhost:43579/backupUT\x00jenkins-hbase3.apache.org:40209/meta:rs-log-ts/1691649081772/Put/seqid=0 2023-08-10T06:31:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-08-10T06:31:29,392 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=112 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/meta/00eae528e68d47598a7cffe1b24c5340 2023-08-10T06:31:29,400 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 00eae528e68d47598a7cffe1b24c5340 2023-08-10T06:31:29,403 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/3a8b36df8daf43aca677b4468de2f4d1 is 218, key is session:backup_1691649077525/session:context/1691649088791/Put/seqid=0 2023-08-10T06:31:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-08-10T06:31:29,502 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region faa733e2577a03b5cd26e4424a6f4087 changed from -1.0 to 0.0, refreshing cache 2023-08-10T06:31:29,807 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=428 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/3a8b36df8daf43aca677b4468de2f4d1 2023-08-10T06:31:29,813 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3a8b36df8daf43aca677b4468de2f4d1 2023-08-10T06:31:29,814 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/meta/00eae528e68d47598a7cffe1b24c5340 as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/00eae528e68d47598a7cffe1b24c5340 2023-08-10T06:31:29,820 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/00eae528e68d47598a7cffe1b24c5340, entries=1, sequenceid=39, filesize=5.3 K 2023-08-10T06:31:29,821 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/3a8b36df8daf43aca677b4468de2f4d1 as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/3a8b36df8daf43aca677b4468de2f4d1 2023-08-10T06:31:29,826 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/3a8b36df8daf43aca677b4468de2f4d1, entries=1, sequenceid=39, filesize=5.2 K 2023-08-10T06:31:29,827 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(3022): Finished flush of dataSize ~540 B/540, heapSize ~1.21 KB/1240, currentSize=0 B/0 for 4cb36c501bd471befb9ec81ac2da937f in 856ms, sequenceid=39, compaction requested=true 2023-08-10T06:31:29,829 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6cbd9d64f8534727b32d2198cce57056 2023-08-10T06:31:29,830 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 00eae528e68d47598a7cffe1b24c5340 2023-08-10T06:31:29,832 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 34d64473585d4371aa81253c6048c0d8 2023-08-10T06:31:29,833 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f6e3a0a30de744a4a3d21c98964e6218 2023-08-10T06:31:29,835 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f15ed284b4d44c27b14fc474f7eaa879 2023-08-10T06:31:29,836 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3a8b36df8daf43aca677b4468de2f4d1 2023-08-10T06:31:29,840 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/recovered.edits/42.seqid, newMaxSeqId=42, maxSeqId=27 2023-08-10T06:31:29,841 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:31:29,842 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1909): Closed backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:29,842 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1622): Region close journal for 4cb36c501bd471befb9ec81ac2da937f: 2023-08-10T06:31:29,843 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(163): Closed 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:29,844 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=57 updating hbase:meta row=4cb36c501bd471befb9ec81ac2da937f, regionState=CLOSED 2023-08-10T06:31:29,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=58, resume processing ppid=57 2023-08-10T06:31:29,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=58, ppid=57, state=SUCCESS; CloseRegionProcedure 4cb36c501bd471befb9ec81ac2da937f, server=jenkins-hbase3.apache.org,40209,1691649026642 in 1.0260 sec 2023-08-10T06:31:29,847 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=57, resume processing ppid=56 2023-08-10T06:31:29,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=57, ppid=56, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, UNASSIGN in 1.0300 sec 2023-08-10T06:31:29,848 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649089848"}]},"ts":"1691649089848"} 2023-08-10T06:31:29,849 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-08-10T06:31:29,851 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-08-10T06:31:29,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=56, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0450 sec 2023-08-10T06:31:29,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=56 2023-08-10T06:31:29,926 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1691649089926 table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649089926 (current time:1691649089926). 2023-08-10T06:31:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:31:29,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1691649089926 VERSION not specified, setting to 2 2023-08-10T06:31:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af0617f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:29,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:29,934 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:43244, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:29,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:31:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=59, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649089926 table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:29,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649089926 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 59 2023-08-10T06:31:29,938 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649089926 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:31:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=59 2023-08-10T06:31:29,939 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649089926 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:31:29,941 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649089926 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:31:30,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=59 2023-08-10T06:31:30,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=59 2023-08-10T06:31:30,349 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649089926 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-08-10T06:31:30,350 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-08-10T06:31:30,350 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-08-10T06:31:30,350 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:31:30,352 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/00eae528e68d47598a7cffe1b24c5340, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056] hfiles 2023-08-10T06:31:30,352 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/00eae528e68d47598a7cffe1b24c5340 2023-08-10T06:31:30,352 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056 2023-08-10T06:31:30,352 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:31:30,353 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/34d64473585d4371aa81253c6048c0d8, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/3a8b36df8daf43aca677b4468de2f4d1, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f15ed284b4d44c27b14fc474f7eaa879, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f6e3a0a30de744a4a3d21c98964e6218] hfiles 2023-08-10T06:31:30,353 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/4): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/34d64473585d4371aa81253c6048c0d8 2023-08-10T06:31:30,354 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/4): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/3a8b36df8daf43aca677b4468de2f4d1 2023-08-10T06:31:30,354 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/4): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f15ed284b4d44c27b14fc474f7eaa879 2023-08-10T06:31:30,354 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (4/4): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f6e3a0a30de744a4a3d21c98964e6218 2023-08-10T06:31:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=59 2023-08-10T06:31:30,759 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 4cb36c501bd471befb9ec81ac2da937f, NAME => 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-08-10T06:31:30,760 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649089926 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:31:30,760 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649089926 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:31:30,760 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1691649089926 2023-08-10T06:31:30,761 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1691649089926 2023-08-10T06:31:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=59 2023-08-10T06:31:31,172 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649089926 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:31:31,180 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649089926 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:31:31,181 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1691649089926 to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1691649089926 2023-08-10T06:31:32,000 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=59, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649089926 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:31:32,000 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649089926 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 59 2023-08-10T06:31:32,001 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=59, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=59, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649089926 table=backup:system type=FLUSH ttl=0 } in 2.0640 sec 2023-08-10T06:31:32,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=59 2023-08-10T06:31:32,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=60, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691649077562 type: FLUSH version: 2 ttl: 0 ) 2023-08-10T06:31:32,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-08-10T06:31:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-08-10T06:31:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-08-10T06:31:32,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-08-10T06:31:32,467 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-08-10T06:31:32,467 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-08-10T06:31:32,468 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:32,471 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691649077562 type: FLUSH version: 2 ttl: 0 2023-08-10T06:31:32,471 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system 2023-08-10T06:31:32,474 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-08-10T06:31:32,474 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:32,480 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/00eae528e68d47598a7cffe1b24c5340 to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/archive/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/00eae528e68d47598a7cffe1b24c5340 2023-08-10T06:31:32,480 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:31:32,485 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/3a8b36df8daf43aca677b4468de2f4d1 to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/archive/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/3a8b36df8daf43aca677b4468de2f4d1 2023-08-10T06:31:32,485 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:31:32,486 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691649077562 type: FLUSH version: 2 ttl: 0 2023-08-10T06:31:32,487 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691649092486"}]},"ts":"9223372036854775807"} 2023-08-10T06:31:32,489 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:31:32,489 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 4cb36c501bd471befb9ec81ac2da937f, NAME => 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:31:32,489 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1691649092487"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649092487"}]},"ts":"1691649092487"} 2023-08-10T06:31:32,491 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:31:32,491 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-08-10T06:31:32,491 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 4cb36c501bd471befb9ec81ac2da937f, NAME => 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:31:32,491 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-08-10T06:31:32,493 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=60, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691649077562 type: FLUSH version: 2 ttl: 0 ) in 438 msec 2023-08-10T06:31:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=60 2023-08-10T06:31:32,660 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1691649089926" 2023-08-10T06:31:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1691649089926 2023-08-10T06:31:32,664 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster$11(2630): Client=jenkins//172.31.12.81 enable backup:system 2023-08-10T06:31:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=61, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-08-10T06:31:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=61 2023-08-10T06:31:32,668 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-08-10T06:31:32,668 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649092668"}]},"ts":"1691649092668"} 2023-08-10T06:31:32,669 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-08-10T06:31:32,672 DEBUG [PEWorker-5 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-08-10T06:31:32,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, ASSIGN}] 2023-08-10T06:31:32,674 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, ASSIGN 2023-08-10T06:31:32,674 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-08-10T06:31:32,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=61 2023-08-10T06:31:32,826 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=62 updating hbase:meta row=4cb36c501bd471befb9ec81ac2da937f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:32,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; OpenRegionProcedure 4cb36c501bd471befb9ec81ac2da937f, server=jenkins-hbase3.apache.org,40209,1691649026642}] 2023-08-10T06:31:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=61 2023-08-10T06:31:32,983 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(130): Open backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:32,983 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7254): Opening region: {ENCODED => 4cb36c501bd471befb9ec81ac2da937f, NAME => 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:31:32,983 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:31:32,983 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:32,983 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(888): Instantiated backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:31:32,983 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7296): checking encryption for 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:32,983 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7299): checking classloading for 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:32,985 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:32,986 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4cb36c501bd471befb9ec81ac2da937f columnFamilyName meta 2023-08-10T06:31:32,986 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:31:32,992 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056 2023-08-10T06:31:32,992 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.HStore(324): Store=4cb36c501bd471befb9ec81ac2da937f/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:31:32,992 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:32,993 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4cb36c501bd471befb9ec81ac2da937f columnFamilyName session 2023-08-10T06:31:32,993 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:31:32,999 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/34d64473585d4371aa81253c6048c0d8 2023-08-10T06:31:33,004 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f15ed284b4d44c27b14fc474f7eaa879 2023-08-10T06:31:33,009 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f6e3a0a30de744a4a3d21c98964e6218 2023-08-10T06:31:33,010 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.HStore(324): Store=4cb36c501bd471befb9ec81ac2da937f/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:31:33,011 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:33,012 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:33,014 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:31:33,015 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1076): writing seq id for 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:33,016 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): Opened 4cb36c501bd471befb9ec81ac2da937f; next sequenceid=43; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11584871680, jitterRate=0.0789252519607544}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:31:33,016 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(992): Region open journal for 4cb36c501bd471befb9ec81ac2da937f: 2023-08-10T06:31:33,017 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f., pid=63, masterSystemTime=1691649092979 2023-08-10T06:31:33,019 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.CompactSplit(403): Add compact mark for store 4cb36c501bd471befb9ec81ac2da937f:session, priority=-2147483648, current under compaction store size is 1 2023-08-10T06:31:33,021 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-08-10T06:31:33,021 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-08-10T06:31:33,022 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:33,022 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(158): Opened backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:33,023 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=62 updating hbase:meta row=4cb36c501bd471befb9ec81ac2da937f, regionState=OPEN, openSeqNum=43, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:33,024 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 16169 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-08-10T06:31:33,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=63, resume processing ppid=62 2023-08-10T06:31:33,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=63, ppid=62, state=SUCCESS; OpenRegionProcedure 4cb36c501bd471befb9ec81ac2da937f, server=jenkins-hbase3.apache.org,40209,1691649026642 in 197 msec 2023-08-10T06:31:33,026 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] regionserver.HStore(1534): 4cb36c501bd471befb9ec81ac2da937f/session is initiating minor compaction (all files) 2023-08-10T06:31:33,026 INFO [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 4cb36c501bd471befb9ec81ac2da937f/session in backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:33,026 INFO [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/34d64473585d4371aa81253c6048c0d8, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f6e3a0a30de744a4a3d21c98964e6218, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f15ed284b4d44c27b14fc474f7eaa879] into tmpdir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp, totalSize=15.8 K 2023-08-10T06:31:33,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=62, resume processing ppid=61 2023-08-10T06:31:33,027 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=62, ppid=61, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, ASSIGN in 354 msec 2023-08-10T06:31:33,028 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] compactions.Compactor(223): Compacting 34d64473585d4371aa81253c6048c0d8, keycount=1, bloomtype=ROW, size=4.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1691649049291 2023-08-10T06:31:33,028 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649093028"}]},"ts":"1691649093028"} 2023-08-10T06:31:33,028 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] compactions.Compactor(223): Compacting f6e3a0a30de744a4a3d21c98964e6218, keycount=3, bloomtype=ROW, size=5.7 K, encoding=NONE, compression=NONE, seqNum=20, earliestPutTs=1691649068420 2023-08-10T06:31:33,029 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] compactions.Compactor(223): Compacting f15ed284b4d44c27b14fc474f7eaa879, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1691649077442 2023-08-10T06:31:33,029 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-08-10T06:31:33,031 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-08-10T06:31:33,032 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=61, state=SUCCESS; EnableTableProcedure table=backup:system in 367 msec 2023-08-10T06:31:33,054 INFO [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cb36c501bd471befb9ec81ac2da937f#session#compaction#10 average throughput is 0.95 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-08-10T06:31:33,055 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/e0b86ba5e37b4d82b985f61689fb9ccc is 459, key is session:backup_1691649045860/session:context/1691649068420/Put/seqid=0 2023-08-10T06:31:33,068 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e0b86ba5e37b4d82b985f61689fb9ccc 2023-08-10T06:31:33,069 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/e0b86ba5e37b4d82b985f61689fb9ccc as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/e0b86ba5e37b4d82b985f61689fb9ccc 2023-08-10T06:31:33,084 INFO [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 4cb36c501bd471befb9ec81ac2da937f/session of 4cb36c501bd471befb9ec81ac2da937f into e0b86ba5e37b4d82b985f61689fb9ccc(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-08-10T06:31:33,084 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 4cb36c501bd471befb9ec81ac2da937f: 2023-08-10T06:31:33,084 INFO [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f., storeName=4cb36c501bd471befb9ec81ac2da937f/session, priority=13, startTime=1691649093019; duration=0sec 2023-08-10T06:31:33,085 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-08-10T06:31:33,085 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cb36c501bd471befb9ec81ac2da937f:session 2023-08-10T06:31:33,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=61 2023-08-10T06:31:33,270 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-08-10T06:31:33,270 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:31:33,275 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:31:33,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:31:33,280 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:31:33,280 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: INCREMENTAL_COPY 2023-08-10T06:31:33,281 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:43579/backupUT/backup_1691649077525/ns2/test-16916490353991 done. 2023-08-10T06:31:33,282 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(208): hdfs://localhost:43579/backupUT/backup_1691649077525/ns2 is empty, remove it. 2023-08-10T06:31:33,283 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:43579/backupUT/backup_1691649077525/default/test-1691649035399 done. 2023-08-10T06:31:33,284 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(208): hdfs://localhost:43579/backupUT/backup_1691649077525/default is empty, remove it. 2023-08-10T06:31:33,285 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:31:33,290 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(246): Backup backup_1691649077525 failed. Backup session finished. Status: FAILURE 2023-08-10T06:31:33,291 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:33,291 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:137) ~[test-classes/:?] ... 41 more 2023-08-10T06:31:33,295 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 3 2023-08-10T06:31:33,313 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78c5007d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:33,315 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:31:33,316 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40908, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:31:33,319 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:33,321 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49126, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:33,362 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:31:33,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:33,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649093365 (current time:1691649093365). 2023-08-10T06:31:33,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:31:33,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:31:33,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c00257, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:33,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:33,374 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49128, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:33,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:33,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:31:33,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:33,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-08-10T06:31:33,377 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:31:33,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:31:33,378 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:31:33,379 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:31:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:31:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:31:33,789 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:31:33,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 4cb36c501bd471befb9ec81ac2da937f}] 2023-08-10T06:31:33,791 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:33,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40209 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=65 2023-08-10T06:31:33,942 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:33,943 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2819): Flushing 4cb36c501bd471befb9ec81ac2da937f 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-08-10T06:31:33,946 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/255f97ba08f447aca89d284e51e6f514 is 256, key is session:backup_1691649077525/session:context/1691649093284/Put/seqid=0 2023-08-10T06:31:33,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:31:34,351 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/255f97ba08f447aca89d284e51e6f514 2023-08-10T06:31:34,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 255f97ba08f447aca89d284e51e6f514 2023-08-10T06:31:34,358 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/255f97ba08f447aca89d284e51e6f514 as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/255f97ba08f447aca89d284e51e6f514 2023-08-10T06:31:34,363 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/255f97ba08f447aca89d284e51e6f514, entries=2, sequenceid=49, filesize=5.2 K 2023-08-10T06:31:34,364 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for 4cb36c501bd471befb9ec81ac2da937f in 421ms, sequenceid=49, compaction requested=false 2023-08-10T06:31:34,364 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2520): Flush status journal for 4cb36c501bd471befb9ec81ac2da937f: 2023-08-10T06:31:34,365 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. for snapshot_backup_system completed. 2023-08-10T06:31:34,365 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:31:34,365 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:31:34,365 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056] hfiles 2023-08-10T06:31:34,365 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056 for snapshot=snapshot_backup_system 2023-08-10T06:31:34,365 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/e0b86ba5e37b4d82b985f61689fb9ccc, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/255f97ba08f447aca89d284e51e6f514] hfiles 2023-08-10T06:31:34,365 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/e0b86ba5e37b4d82b985f61689fb9ccc for snapshot=snapshot_backup_system 2023-08-10T06:31:34,365 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/255f97ba08f447aca89d284e51e6f514 for snapshot=snapshot_backup_system 2023-08-10T06:31:34,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:31:34,770 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:34,771 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=65 2023-08-10T06:31:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster(3994): Remote procedure done, pid=65 2023-08-10T06:31:34,771 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:34,773 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:34,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=65, resume processing ppid=64 2023-08-10T06:31:34,776 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:31:34,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=65, ppid=64, state=SUCCESS; SnapshotRegionProcedure 4cb36c501bd471befb9ec81ac2da937f in 984 msec 2023-08-10T06:31:34,776 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:31:34,777 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:31:34,777 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:31:34,778 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:31:35,195 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:31:35,202 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:31:35,202 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:31:35,392 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:31:35,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:31:36,021 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:31:36,021 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-08-10T06:31:36,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.6460 sec 2023-08-10T06:31:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=64 2023-08-10T06:31:37,484 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(115): Backup backup_1691649093341 started at 1691649097484. 2023-08-10T06:31:37,486 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(119): Backup session backup_1691649093341 has been started. 2023-08-10T06:31:37,487 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] backup.TestBackupBase$IncrementalTableBackupClientForTest(128): For incremental backup, current table set is [ns2:test-16916490353991, test-1691649035399] 2023-08-10T06:31:37,490 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(76): StartCode 1691649030774for backupID backup_1691649093341 2023-08-10T06:31:37,490 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-08-10T06:31:37,490 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:31:37,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:31:37,491 INFO [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:31:37,491 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:31:37,492 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:31:37,492 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:31:37,492 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:31:37,494 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:37,494 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:31:37,494 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:31:37,494 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:31:37,494 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:31:37,495 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:31:37,495 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:31:37,495 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:31:37,495 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:31:37,495 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:31:37,495 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:31:37,495 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:31:37,496 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:31:37,496 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:31:37,496 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,40209,1691649026642' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:31:37,498 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:37,498 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:37,498 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:31:37,498 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:31:37,499 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:31:37,499 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:31:37,499 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:31:37,499 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:37,499 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:37,500 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:31:37,500 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,40209,1691649026642' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:31:37,500 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@4eeb8884[Count = 0] remaining members to acquire global barrier 2023-08-10T06:31:37,500 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:31:37,500 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:37,502 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:37,502 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:37,502 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:31:37,502 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:31:37,502 DEBUG [rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:37,502 INFO [rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691649081737 highest: 1691649081737 on jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:37,502 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1:(num 1691649081704) roll requested 2023-08-10T06:31:37,503 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:31:37,514 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649097503, exclude list is [], retry=0 2023-08-10T06:31:37,517 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK] 2023-08-10T06:31:37,519 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649081704 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649097503 2023-08-10T06:31:37,519 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK]] 2023-08-10T06:31:37,519 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649081704 is not closed yet, will try archiving it next time 2023-08-10T06:31:37,519 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta:.meta(num 1691649081721) roll requested 2023-08-10T06:31:37,522 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(655): complete file /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649081704 not finished, retry = 0 2023-08-10T06:31:37,531 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649097519.meta, exclude list is [], retry=0 2023-08-10T06:31:37,534 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK] 2023-08-10T06:31:37,536 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649081721.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649097519.meta 2023-08-10T06:31:37,536 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK]] 2023-08-10T06:31:37,536 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649081721.meta is not closed yet, will try archiving it next time 2023-08-10T06:31:37,536 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0:(num 1691649081737) roll requested 2023-08-10T06:31:37,547 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649097536, exclude list is [], retry=0 2023-08-10T06:31:37,550 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK] 2023-08-10T06:31:37,551 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649081737 with entries=13, filesize=3.56 KB; new WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649097536 2023-08-10T06:31:37,551 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK]] 2023-08-10T06:31:37,551 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649081737 is not closed yet, will try archiving it next time 2023-08-10T06:31:37,563 DEBUG [rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-08-10T06:31:37,563 INFO [rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691649097536 on jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:37,571 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:31:37,571 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,40209,1691649026642' in zk 2023-08-10T06:31:37,574 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:37,574 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:31:37,574 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:31:37,574 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:31:37,574 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:31:37,574 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:31:37,575 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:31:37,575 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:31:37,576 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:37,576 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:37,576 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:31:37,577 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:37,577 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:37,578 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,40209,1691649026642': 2023-08-10T06:31:37,578 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,40209,1691649026642' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:31:37,578 INFO [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:31:37,578 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:31:37,578 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:31:37,578 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:31:37,578 INFO [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:31:37,580 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:31:37,580 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:31:37,580 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:31:37,580 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:31:37,580 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:31:37,580 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:31:37,580 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:31:37,580 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:31:37,580 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:37,580 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:31:37,581 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:31:37,581 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:37,581 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:37,582 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:31:37,582 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:37,583 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:37,586 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:31:37,586 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:31:37,586 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:31:37,586 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:31:37,586 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:31:37,587 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:31:37,587 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:31:37,623 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649081704 to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/oldWALs/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649081704 2023-08-10T06:31:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:31:37,690 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:40209=1691649030774} newestTimestamps: {jenkins-hbase3.apache.org:40209=1691649081737} 2023-08-10T06:31:37,692 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649029122.meta 2023-08-10T06:31:37,692 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649053525.meta 2023-08-10T06:31:37,692 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649081721.meta 2023-08-10T06:31:37,692 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649097519.meta 2023-08-10T06:31:37,692 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543 2023-08-10T06:31:37,692 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649081737 2023-08-10T06:31:37,692 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649097536 2023-08-10T06:31:37,692 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 2023-08-10T06:31:37,692 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649097503 2023-08-10T06:31:37,693 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/oldWALs/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649028773 2023-08-10T06:31:37,693 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/oldWALs/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649081704 2023-08-10T06:31:38,103 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16916490353991 target: hdfs://localhost:43579/backupUT/backup_1691649093341/ns2/test-16916490353991 descriptor: 'ns2:test-16916490353991', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:31:38,103 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-08-10T06:31:38,105 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16916490353991 2023-08-10T06:31:38,510 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16916490353991 2023-08-10T06:31:38,920 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(146): Attempting to copy table info for:test-1691649035399 target: hdfs://localhost:43579/backupUT/backup_1691649093341/default/test-1691649035399 descriptor: 'test-1691649035399', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:31:38,920 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-08-10T06:31:38,922 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(151): Starting to write region info for table test-1691649035399 2023-08-10T06:31:39,328 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(158): Finished writing region info for table test-1691649035399 2023-08-10T06:31:39,333 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:43579/backupUT/.tmp/backup_1691649093341 from hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543;hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649081737;hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 2023-08-10T06:31:39,340 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@abd55ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:39,341 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:31:39,342 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:40918, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:31:39,344 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-08-10T06:31:39,344 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16916490353991 2023-08-10T06:31:39,347 DEBUG [hconnection-0x4853a162-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:39,349 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:49132, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:39,350 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16916490353991: ns2:test-16916490353991; 2023-08-10T06:31:39,350 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1691649035399 2023-08-10T06:31:39,352 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1691649035399: test-1691649035399; 2023-08-10T06:31:39,352 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-08-10T06:31:39,352 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/hbase-staging/partitions_013cbadf-9d90-4136-b7de-cf189e2a80ab 2023-08-10T06:31:39,759 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:39,759 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:39,760 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:39,760 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:40,716 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/hadoop-3264863031910268278.jar 2023-08-10T06:31:40,716 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:40,717 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:40,717 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:40,717 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:40,718 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:40,718 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:40,719 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:40,719 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:40,719 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:40,720 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:31:40,721 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:31:40,721 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:31:40,722 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:31:40,722 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:31:40,722 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:31:40,723 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:31:40,723 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:31:40,724 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:31:40,724 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:31:40,725 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:31:40,725 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:31:40,725 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:40,726 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:40,726 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:40,727 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:40,727 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:40,727 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:40,728 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:40,728 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16916490353991,default:test-1691649035399 2023-08-10T06:31:40,728 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:40,729 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:31:41,719 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/hadoop-9096548093857125918.jar 2023-08-10T06:31:41,739 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543 for WAL files 2023-08-10T06:31:41,740 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543 2023-08-10T06:31:41,740 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649081737 for WAL files 2023-08-10T06:31:41,741 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649081737 2023-08-10T06:31:41,741 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 for WAL files 2023-08-10T06:31:41,742 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 2023-08-10T06:31:41,812 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/mapreduce.cluster.local.dir/1691649101778/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:31:41,813 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/mapreduce.cluster.local.dir/1691649101778/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:31:41,872 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543 for hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543 (-9223372036854775808:9223372036854775807) length:18860 2023-08-10T06:31:41,879 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-08-10T06:31:41,879 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-08-10T06:31:41,903 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649081737 for hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649081737 (-9223372036854775808:9223372036854775807) length:3657 2023-08-10T06:31:41,906 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-08-10T06:31:41,906 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-08-10T06:31:41,928 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 for hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 (-9223372036854775808:9223372036854775807) length:1198 2023-08-10T06:31:41,932 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-08-10T06:31:41,932 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-08-10T06:31:41,952 DEBUG [pool-246-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@178eb175, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:41,955 DEBUG [hconnection-0x2c511c0a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:41,957 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56366, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:41,958 DEBUG [pool-246-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:41,960 DEBUG [pool-246-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t20] 2023-08-10T06:31:41,960 DEBUG [pool-246-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-08-10T06:31:41,963 DEBUG [pool-246-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/backupUT/.tmp/backup_1691649093341/_temporary/0/_temporary/attempt_local1812666350_0005_r_000000_0/ns2/test-16916490353991/f/a1a3767591e84eaab735942d5da1f8f0 is 38, key is row-t20/f:q1/1691649068577/Put/seqid=0 2023-08-10T06:31:41,965 WARN [Thread-1143 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:40209] Specified favored nodes: [jenkins-hbase3.apache.org:40209] 2023-08-10T06:31:42,389 DEBUG [pool-246-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c098a25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:42,390 DEBUG [hconnection-0x34121ea5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:42,392 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56370, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:42,393 DEBUG [pool-246-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:42,394 DEBUG [pool-246-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-test-1691649035399-1-0] 2023-08-10T06:31:42,395 DEBUG [pool-246-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-08-10T06:31:42,404 DEBUG [pool-246-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/backupUT/.tmp/backup_1691649093341/_temporary/0/_temporary/attempt_local1812666350_0005_r_000001_0/default/test-1691649035399/f/5b953eb07b8243f28787307b8bee5d18 is 59, key is row-test-1691649035399-1-10/f:q1/1691649068469/Put/seqid=0 2023-08-10T06:31:42,406 WARN [Thread-1153 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:40209] Specified favored nodes: [jenkins-hbase3.apache.org:40209] 2023-08-10T06:31:42,862 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:43579/backupUT 2023-08-10T06:31:42,862 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1691649093341 2023-08-10T06:31:42,862 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-08-10T06:31:42,868 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:43579/backupUT/.tmp/backup_1691649093341, hdfs://localhost:43579/backupUT] 2023-08-10T06:31:43,973 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-08-10T06:31:43,975 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1691649093341 2023-08-10T06:31:43,975 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 16005 bytes copied." 2023-08-10T06:31:43,975 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local1208739012_0006 completed: true true 2023-08-10T06:31:43,978 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=4446446 FILE: Number of bytes written=7503672 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=306899 HDFS: Number of bytes written=2429277 HDFS: Number of read operations=1066 HDFS: Number of large read operations=0 HDFS: Number of write operations=366 Map-Reduce Framework Map input records=10 Map output records=0 Input split bytes=341 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1546125312 File Input Format Counters Bytes Read=2444 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=16005 Bytes Expected=16005 Files Copied=3 DIR_COPY=7 2023-08-10T06:31:43,978 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:43579/backupUT/.tmp/backup_1691649093341 to hdfs://localhost:43579/backupUT finished. 2023-08-10T06:31:43,983 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_3 in testing java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:150) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-08-10T06:31:43,984 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(240): BackupId=backup_1691649093341,startts=1691649097484,failedts=1691649103983,failedphase=INCREMENTAL_COPY,failedmessage=Failed stage stage_3 in testing 2023-08-10T06:31:43,984 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-08-10T06:31:43,988 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable backup:system 2023-08-10T06:31:43,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=66, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-08-10T06:31:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-08-10T06:31:43,993 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649103992"}]},"ts":"1691649103992"} 2023-08-10T06:31:43,994 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-08-10T06:31:43,996 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-08-10T06:31:43,996 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, UNASSIGN}] 2023-08-10T06:31:43,999 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, UNASSIGN 2023-08-10T06:31:44,000 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=4cb36c501bd471befb9ec81ac2da937f, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:44,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; CloseRegionProcedure 4cb36c501bd471befb9ec81ac2da937f, server=jenkins-hbase3.apache.org,40209,1691649026642}] 2023-08-10T06:31:44,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-08-10T06:31:44,153 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] handler.UnassignRegionHandler(119): Close 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:44,153 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1668): Closing 4cb36c501bd471befb9ec81ac2da937f, disabling compactions & flushes 2023-08-10T06:31:44,153 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1690): Closing region backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:44,153 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:44,153 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. after waiting 0 ms 2023-08-10T06:31:44,153 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:44,153 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(2819): Flushing 4cb36c501bd471befb9ec81ac2da937f 2/2 column families, dataSize=872 B heapSize=1.71 KB 2023-08-10T06:31:44,166 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/meta/5320793b141d4be284a05731af7c3dd3 is 171, key is trslm:hdfs://localhost:43579/backupUT\x00ns2:test-16916490353991/meta:log-roll-map/1691649103982/Put/seqid=0 2023-08-10T06:31:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-08-10T06:31:44,571 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=444 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/meta/5320793b141d4be284a05731af7c3dd3 2023-08-10T06:31:44,576 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 5320793b141d4be284a05731af7c3dd3 2023-08-10T06:31:44,578 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/dc37460e580c434ab0a7bc8d1f81b171 is 218, key is session:backup_1691649093341/session:context/1691649103974/Put/seqid=0 2023-08-10T06:31:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-08-10T06:31:44,983 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=428 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/dc37460e580c434ab0a7bc8d1f81b171 2023-08-10T06:31:44,988 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: dc37460e580c434ab0a7bc8d1f81b171 2023-08-10T06:31:44,989 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/meta/5320793b141d4be284a05731af7c3dd3 as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/5320793b141d4be284a05731af7c3dd3 2023-08-10T06:31:44,994 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/5320793b141d4be284a05731af7c3dd3, entries=3, sequenceid=56, filesize=5.6 K 2023-08-10T06:31:44,995 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/dc37460e580c434ab0a7bc8d1f81b171 as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/dc37460e580c434ab0a7bc8d1f81b171 2023-08-10T06:31:45,001 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/dc37460e580c434ab0a7bc8d1f81b171, entries=1, sequenceid=56, filesize=5.2 K 2023-08-10T06:31:45,003 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(3022): Finished flush of dataSize ~872 B/872, heapSize ~1.68 KB/1720, currentSize=0 B/0 for 4cb36c501bd471befb9ec81ac2da937f in 850ms, sequenceid=56, compaction requested=true 2023-08-10T06:31:45,005 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6cbd9d64f8534727b32d2198cce57056 2023-08-10T06:31:45,006 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 5320793b141d4be284a05731af7c3dd3 2023-08-10T06:31:45,008 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 34d64473585d4371aa81253c6048c0d8 2023-08-10T06:31:45,009 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f6e3a0a30de744a4a3d21c98964e6218 2023-08-10T06:31:45,011 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f15ed284b4d44c27b14fc474f7eaa879 2023-08-10T06:31:45,012 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/34d64473585d4371aa81253c6048c0d8, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f6e3a0a30de744a4a3d21c98964e6218, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f15ed284b4d44c27b14fc474f7eaa879] to archive 2023-08-10T06:31:45,013 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-08-10T06:31:45,014 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 34d64473585d4371aa81253c6048c0d8 2023-08-10T06:31:45,016 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/34d64473585d4371aa81253c6048c0d8 to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/archive/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/34d64473585d4371aa81253c6048c0d8 2023-08-10T06:31:45,017 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f6e3a0a30de744a4a3d21c98964e6218 2023-08-10T06:31:45,018 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f6e3a0a30de744a4a3d21c98964e6218 to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/archive/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f6e3a0a30de744a4a3d21c98964e6218 2023-08-10T06:31:45,019 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f15ed284b4d44c27b14fc474f7eaa879 2023-08-10T06:31:45,020 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f15ed284b4d44c27b14fc474f7eaa879 to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/archive/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/f15ed284b4d44c27b14fc474f7eaa879 2023-08-10T06:31:45,050 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e0b86ba5e37b4d82b985f61689fb9ccc 2023-08-10T06:31:45,052 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 255f97ba08f447aca89d284e51e6f514 2023-08-10T06:31:45,053 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: dc37460e580c434ab0a7bc8d1f81b171 2023-08-10T06:31:45,057 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/recovered.edits/59.seqid, newMaxSeqId=59, maxSeqId=42 2023-08-10T06:31:45,058 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:31:45,059 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1909): Closed backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:45,059 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] regionserver.HRegion(1622): Region close journal for 4cb36c501bd471befb9ec81ac2da937f: 2023-08-10T06:31:45,060 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=68}] handler.UnassignRegionHandler(163): Closed 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:45,061 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=67 updating hbase:meta row=4cb36c501bd471befb9ec81ac2da937f, regionState=CLOSED 2023-08-10T06:31:45,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=68, resume processing ppid=67 2023-08-10T06:31:45,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=68, ppid=67, state=SUCCESS; CloseRegionProcedure 4cb36c501bd471befb9ec81ac2da937f, server=jenkins-hbase3.apache.org,40209,1691649026642 in 1.0610 sec 2023-08-10T06:31:45,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=67, resume processing ppid=66 2023-08-10T06:31:45,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=67, ppid=66, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, UNASSIGN in 1.0670 sec 2023-08-10T06:31:45,065 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649105065"}]},"ts":"1691649105065"} 2023-08-10T06:31:45,066 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-08-10T06:31:45,068 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-08-10T06:31:45,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=66, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0790 sec 2023-08-10T06:31:45,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=66 2023-08-10T06:31:45,105 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1691649105104 table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649105105 (current time:1691649105105). 2023-08-10T06:31:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:31:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1691649105104 VERSION not specified, setting to 2 2023-08-10T06:31:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f115f23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:45,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:45,113 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56374, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:45,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:45,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:31:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=69, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649105104 table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649105104 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 69 2023-08-10T06:31:45,116 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649105104 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:31:45,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=69 2023-08-10T06:31:45,117 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649105104 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:31:45,119 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649105104 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:31:45,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=69 2023-08-10T06:31:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=69 2023-08-10T06:31:45,526 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649105104 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-08-10T06:31:45,527 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-08-10T06:31:45,527 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-08-10T06:31:45,527 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:31:45,529 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/5320793b141d4be284a05731af7c3dd3, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056] hfiles 2023-08-10T06:31:45,529 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/5320793b141d4be284a05731af7c3dd3 2023-08-10T06:31:45,529 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056 2023-08-10T06:31:45,529 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:31:45,530 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/255f97ba08f447aca89d284e51e6f514, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/dc37460e580c434ab0a7bc8d1f81b171, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/e0b86ba5e37b4d82b985f61689fb9ccc] hfiles 2023-08-10T06:31:45,530 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/255f97ba08f447aca89d284e51e6f514 2023-08-10T06:31:45,530 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/dc37460e580c434ab0a7bc8d1f81b171 2023-08-10T06:31:45,530 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/e0b86ba5e37b4d82b985f61689fb9ccc 2023-08-10T06:31:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=69 2023-08-10T06:31:45,936 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 4cb36c501bd471befb9ec81ac2da937f, NAME => 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-08-10T06:31:45,937 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649105104 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:31:45,938 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649105104 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:31:45,938 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1691649105104 2023-08-10T06:31:45,939 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1691649105104 2023-08-10T06:31:46,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=69 2023-08-10T06:31:46,349 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649105104 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:31:46,357 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649105104 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:31:46,357 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1691649105104 to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1691649105104 2023-08-10T06:31:47,174 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=69, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649105104 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:31:47,174 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649105104 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 69 2023-08-10T06:31:47,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=69, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=69, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649105104 table=backup:system type=FLUSH ttl=0 } in 2.0600 sec 2023-08-10T06:31:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=69 2023-08-10T06:31:47,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=70, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691649093365 type: FLUSH version: 2 ttl: 0 ) 2023-08-10T06:31:47,230 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-08-10T06:31:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-08-10T06:31:47,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-08-10T06:31:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-08-10T06:31:47,644 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-08-10T06:31:47,644 INFO [PEWorker-3 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-08-10T06:31:47,645 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:47,648 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691649093365 type: FLUSH version: 2 ttl: 0 2023-08-10T06:31:47,648 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system 2023-08-10T06:31:47,651 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-08-10T06:31:47,651 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:47,656 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/5320793b141d4be284a05731af7c3dd3 to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/archive/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/5320793b141d4be284a05731af7c3dd3 2023-08-10T06:31:47,656 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:31:47,659 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/dc37460e580c434ab0a7bc8d1f81b171 to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/archive/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/dc37460e580c434ab0a7bc8d1f81b171 2023-08-10T06:31:47,659 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:31:47,659 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691649093365 type: FLUSH version: 2 ttl: 0 2023-08-10T06:31:47,660 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691649107660"}]},"ts":"9223372036854775807"} 2023-08-10T06:31:47,662 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:31:47,662 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 4cb36c501bd471befb9ec81ac2da937f, NAME => 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:31:47,662 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1691649107661"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649107661"}]},"ts":"1691649107661"} 2023-08-10T06:31:47,663 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:31:47,664 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-08-10T06:31:47,664 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 4cb36c501bd471befb9ec81ac2da937f, NAME => 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:31:47,664 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-08-10T06:31:47,665 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=70, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691649093365 type: FLUSH version: 2 ttl: 0 ) in 435 msec 2023-08-10T06:31:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=70 2023-08-10T06:31:47,836 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1691649105104" 2023-08-10T06:31:47,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1691649105104 2023-08-10T06:31:47,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster$11(2630): Client=jenkins//172.31.12.81 enable backup:system 2023-08-10T06:31:47,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=71, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-08-10T06:31:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=71 2023-08-10T06:31:47,844 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-08-10T06:31:47,844 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649107844"}]},"ts":"1691649107844"} 2023-08-10T06:31:47,846 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-08-10T06:31:47,848 DEBUG [PEWorker-4 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-08-10T06:31:47,848 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, ASSIGN}] 2023-08-10T06:31:47,850 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, ASSIGN 2023-08-10T06:31:47,851 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-08-10T06:31:47,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=71 2023-08-10T06:31:48,002 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=72 updating hbase:meta row=4cb36c501bd471befb9ec81ac2da937f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:48,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE; OpenRegionProcedure 4cb36c501bd471befb9ec81ac2da937f, server=jenkins-hbase3.apache.org,40209,1691649026642}] 2023-08-10T06:31:48,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=71 2023-08-10T06:31:48,159 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(130): Open backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:48,159 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7254): Opening region: {ENCODED => 4cb36c501bd471befb9ec81ac2da937f, NAME => 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:31:48,159 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:31:48,159 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:48,159 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(888): Instantiated backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:31:48,159 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7296): checking encryption for 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:48,160 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7299): checking classloading for 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:48,161 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:48,162 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4cb36c501bd471befb9ec81ac2da937f columnFamilyName meta 2023-08-10T06:31:48,162 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:31:48,167 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056 2023-08-10T06:31:48,167 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.HStore(324): Store=4cb36c501bd471befb9ec81ac2da937f/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:31:48,168 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:48,169 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4cb36c501bd471befb9ec81ac2da937f columnFamilyName session 2023-08-10T06:31:48,169 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:31:48,174 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/255f97ba08f447aca89d284e51e6f514 2023-08-10T06:31:48,178 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/e0b86ba5e37b4d82b985f61689fb9ccc 2023-08-10T06:31:48,178 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.HStore(324): Store=4cb36c501bd471befb9ec81ac2da937f/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:31:48,179 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:48,180 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:48,181 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:31:48,183 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1076): writing seq id for 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:48,183 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): Opened 4cb36c501bd471befb9ec81ac2da937f; next sequenceid=60; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11647128960, jitterRate=0.08472341299057007}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:31:48,184 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(992): Region open journal for 4cb36c501bd471befb9ec81ac2da937f: 2023-08-10T06:31:48,185 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f., pid=73, masterSystemTime=1691649108156 2023-08-10T06:31:48,186 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:48,186 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(158): Opened backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:48,187 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=72 updating hbase:meta row=4cb36c501bd471befb9ec81ac2da937f, regionState=OPEN, openSeqNum=60, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:48,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=73, resume processing ppid=72 2023-08-10T06:31:48,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=73, ppid=72, state=SUCCESS; OpenRegionProcedure 4cb36c501bd471befb9ec81ac2da937f, server=jenkins-hbase3.apache.org,40209,1691649026642 in 184 msec 2023-08-10T06:31:48,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=72, resume processing ppid=71 2023-08-10T06:31:48,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=72, ppid=71, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, ASSIGN in 342 msec 2023-08-10T06:31:48,192 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649108191"}]},"ts":"1691649108191"} 2023-08-10T06:31:48,193 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-08-10T06:31:48,195 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-08-10T06:31:48,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=71, state=SUCCESS; EnableTableProcedure table=backup:system in 355 msec 2023-08-10T06:31:48,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=71 2023-08-10T06:31:48,447 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-08-10T06:31:48,447 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:31:48,452 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:31:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:31:48,455 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:31:48,455 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: INCREMENTAL_COPY 2023-08-10T06:31:48,456 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:43579/backupUT/backup_1691649093341/ns2/test-16916490353991 done. 2023-08-10T06:31:48,457 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(208): hdfs://localhost:43579/backupUT/backup_1691649093341/ns2 is empty, remove it. 2023-08-10T06:31:48,457 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:43579/backupUT/backup_1691649093341/default/test-1691649035399 done. 2023-08-10T06:31:48,458 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(208): hdfs://localhost:43579/backupUT/backup_1691649093341/default is empty, remove it. 2023-08-10T06:31:48,459 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:31:48,465 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(246): Backup backup_1691649093341 failed. Backup session finished. Status: FAILURE 2023-08-10T06:31:48,465 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:48,465 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:150) ~[test-classes/:?] ... 41 more 2023-08-10T06:31:48,471 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] backup.TestIncrementalBackupWithFailures(136): Running stage 4 2023-08-10T06:31:48,487 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38443cb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:48,488 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:31:48,489 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:37578, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:31:48,492 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:48,494 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56386, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:48,535 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(610): Start new backup exclusive operation 2023-08-10T06:31:48,539 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649108539 (current time:1691649108539). 2023-08-10T06:31:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:31:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-08-10T06:31:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@320e16ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:48,548 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:56392, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:31:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=74, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:31:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 74 2023-08-10T06:31:48,551 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:31:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=74 2023-08-10T06:31:48,552 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:31:48,553 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:31:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=74 2023-08-10T06:31:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=74 2023-08-10T06:31:48,962 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-08-10T06:31:48,962 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE; SnapshotRegionProcedure 4cb36c501bd471befb9ec81ac2da937f}] 2023-08-10T06:31:48,964 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=75, ppid=74, state=RUNNABLE; SnapshotRegionProcedure 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:49,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40209 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=75 2023-08-10T06:31:49,116 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:49,117 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HRegion(2819): Flushing 4cb36c501bd471befb9ec81ac2da937f 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-08-10T06:31:49,136 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/a8091fba6ddb4ddd92b3536f0938b8be is 256, key is session:backup_1691649093341/session:context/1691649108459/Put/seqid=0 2023-08-10T06:31:49,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=74 2023-08-10T06:31:49,541 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/a8091fba6ddb4ddd92b3536f0938b8be 2023-08-10T06:31:49,546 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a8091fba6ddb4ddd92b3536f0938b8be 2023-08-10T06:31:49,547 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/a8091fba6ddb4ddd92b3536f0938b8be as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/a8091fba6ddb4ddd92b3536f0938b8be 2023-08-10T06:31:49,552 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/a8091fba6ddb4ddd92b3536f0938b8be, entries=2, sequenceid=65, filesize=5.2 K 2023-08-10T06:31:49,553 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for 4cb36c501bd471befb9ec81ac2da937f in 436ms, sequenceid=65, compaction requested=true 2023-08-10T06:31:49,553 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-08-10T06:31:49,553 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.HRegion(2520): Flush status journal for 4cb36c501bd471befb9ec81ac2da937f: 2023-08-10T06:31:49,553 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. for snapshot_backup_system completed. 2023-08-10T06:31:49,553 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.' region-info for snapshot=snapshot_backup_system 2023-08-10T06:31:49,553 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-08-10T06:31:49,553 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056] hfiles 2023-08-10T06:31:49,554 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056 for snapshot=snapshot_backup_system 2023-08-10T06:31:49,554 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/e0b86ba5e37b4d82b985f61689fb9ccc, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/255f97ba08f447aca89d284e51e6f514, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/a8091fba6ddb4ddd92b3536f0938b8be] hfiles 2023-08-10T06:31:49,554 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/e0b86ba5e37b4d82b985f61689fb9ccc for snapshot=snapshot_backup_system 2023-08-10T06:31:49,554 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/255f97ba08f447aca89d284e51e6f514 for snapshot=snapshot_backup_system 2023-08-10T06:31:49,554 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/a8091fba6ddb4ddd92b3536f0938b8be for snapshot=snapshot_backup_system 2023-08-10T06:31:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=74 2023-08-10T06:31:49,959 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:49,959 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=75 2023-08-10T06:31:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster(3994): Remote procedure done, pid=75 2023-08-10T06:31:49,959 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:49,961 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=75, ppid=74, state=RUNNABLE; SnapshotRegionProcedure 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:49,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=75, resume processing ppid=74 2023-08-10T06:31:49,964 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-08-10T06:31:49,964 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=75, ppid=74, state=SUCCESS; SnapshotRegionProcedure 4cb36c501bd471befb9ec81ac2da937f in 1.0000 sec 2023-08-10T06:31:49,964 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:31:49,965 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:31:49,965 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-08-10T06:31:49,966 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/snapshot_backup_system 2023-08-10T06:31:50,392 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:31:50,401 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:31:50,401 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/snapshot_backup_system 2023-08-10T06:31:50,611 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:31:50,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=74 2023-08-10T06:31:51,230 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=74, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:31:51,230 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 74 2023-08-10T06:31:51,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=74, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=74, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.6810 sec 2023-08-10T06:31:52,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=74 2023-08-10T06:31:52,658 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(115): Backup backup_1691649108515 started at 1691649112658. 2023-08-10T06:31:52,660 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(119): Backup session backup_1691649108515 has been started. 2023-08-10T06:31:52,661 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] backup.TestBackupBase$IncrementalTableBackupClientForTest(128): For incremental backup, current table set is [ns2:test-16916490353991, test-1691649035399] 2023-08-10T06:31:52,663 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(76): StartCode 1691649030774for backupID backup_1691649108515 2023-08-10T06:31:52,663 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(86): Execute roll log procedure for incremental backup ... 2023-08-10T06:31:52,664 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(946): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-08-10T06:31:52,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-08-10T06:31:52,664 INFO [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-08-10T06:31:52,664 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-08-10T06:31:52,666 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-08-10T06:31:52,666 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-08-10T06:31:52,666 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:31:52,667 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:52,668 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:31:52,668 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:31:52,668 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-08-10T06:31:52,668 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-08-10T06:31:52,668 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-08-10T06:31:52,669 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-08-10T06:31:52,669 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-08-10T06:31:52,669 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-08-10T06:31:52,669 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-08-10T06:31:52,669 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-08-10T06:31:52,669 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-08-10T06:31:52,670 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-08-10T06:31:52,670 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-08-10T06:31:52,670 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,40209,1691649026642' joining acquired barrier for procedure (rolllog) in zk 2023-08-10T06:31:52,672 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:52,672 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:52,672 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:31:52,672 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:31:52,672 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-08-10T06:31:52,672 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:31:52,673 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:31:52,673 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:52,673 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:52,673 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:31:52,674 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,40209,1691649026642' joining acquired barrier for procedure 'rolllog' on coordinator 2023-08-10T06:31:52,674 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@19b276f[Count = 0] remaining members to acquire global barrier 2023-08-10T06:31:52,674 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-08-10T06:31:52,674 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:52,676 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:52,676 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-08-10T06:31:52,676 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-08-10T06:31:52,676 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-08-10T06:31:52,676 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-08-10T06:31:52,676 DEBUG [rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:52,676 INFO [rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1691649097536 highest: 1691649097536 on jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:52,676 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1:(num 1691649097503) roll requested 2023-08-10T06:31:52,688 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649112676, exclude list is [], retry=0 2023-08-10T06:31:52,693 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK] 2023-08-10T06:31:52,695 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649097503 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649112676 2023-08-10T06:31:52,695 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK]] 2023-08-10T06:31:52,695 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649097503 is not closed yet, will try archiving it next time 2023-08-10T06:31:52,695 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta:.meta(num 1691649097519) roll requested 2023-08-10T06:31:52,697 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649097503 to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/oldWALs/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649097503 2023-08-10T06:31:52,706 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649112695.meta, exclude list is [], retry=0 2023-08-10T06:31:52,709 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK] 2023-08-10T06:31:52,711 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649097519.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649112695.meta 2023-08-10T06:31:52,711 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK]] 2023-08-10T06:31:52,711 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649097519.meta is not closed yet, will try archiving it next time 2023-08-10T06:31:52,711 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0:(num 1691649097536) roll requested 2023-08-10T06:31:52,722 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(531): When create output stream for /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649112711, exclude list is [], retry=0 2023-08-10T06:31:52,725 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK] 2023-08-10T06:31:52,726 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649097536 with entries=13, filesize=3.55 KB; new WAL /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649112711 2023-08-10T06:31:52,727 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:46635,DS-f50d05d4-03fd-404d-8664-f53b0b5949ce,DISK]] 2023-08-10T06:31:52,727 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649097536 is not closed yet, will try archiving it next time 2023-08-10T06:31:52,727 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649081737 to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/oldWALs/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649081737 2023-08-10T06:31:52,737 DEBUG [rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-08-10T06:31:52,737 INFO [rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1691649112711 on jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:52,744 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-08-10T06:31:52,744 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,40209,1691649026642' in zk 2023-08-10T06:31:52,751 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-08-10T06:31:52,751 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:31:52,751 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:52,751 DEBUG [member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-08-10T06:31:52,752 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:31:52,752 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:31:52,752 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:31:52,752 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:31:52,753 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:52,753 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:52,753 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:31:52,753 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:52,754 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:52,754 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,40209,1691649026642': 2023-08-10T06:31:52,755 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,40209,1691649026642' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-08-10T06:31:52,755 INFO [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-08-10T06:31:52,755 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-08-10T06:31:52,755 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-08-10T06:31:52,755 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-08-10T06:31:52,755 INFO [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-08-10T06:31:52,757 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-08-10T06:31:52,757 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-08-10T06:31:52,757 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-08-10T06:31:52,757 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:31:52,757 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-08-10T06:31:52,758 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-08-10T06:31:52,758 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:31:52,758 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:31:52,758 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-08-10T06:31:52,759 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:52,759 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-08-10T06:31:52,760 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:52,760 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:52,760 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-08-10T06:31:52,761 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-08-10T06:31:52,761 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:52,766 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-08-10T06:31:52,766 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-08-10T06:31:52,766 DEBUG [(jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-08-10T06:31:52,766 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-08-10T06:31:52,767 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-08-10T06:31:52,767 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-08-10T06:31:52,767 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-08-10T06:31:52,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1247): Checking to see if procedure from request:rolllog-proc is done 2023-08-10T06:31:52,870 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(131): In getLogFilesForNewBackup() olderTimestamps: {jenkins-hbase3.apache.org:40209=1691649030774} newestTimestamps: {jenkins-hbase3.apache.org:40209=1691649097536} 2023-08-10T06:31:52,872 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649029122.meta 2023-08-10T06:31:52,872 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649053525.meta 2023-08-10T06:31:52,872 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649081721.meta 2023-08-10T06:31:52,872 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649097519.meta 2023-08-10T06:31:52,872 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager$NewestLogFilter(269): Skip .meta log file: jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta.1691649112695.meta 2023-08-10T06:31:52,872 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543 2023-08-10T06:31:52,872 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649097536 2023-08-10T06:31:52,873 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649112711 2023-08-10T06:31:52,873 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 2023-08-10T06:31:52,873 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalBackupManager(177): currentLogFile: hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649112676 2023-08-10T06:31:52,873 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/oldWALs/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649028773 2023-08-10T06:31:52,873 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/oldWALs/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649081737 2023-08-10T06:31:52,874 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/oldWALs/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649081704 2023-08-10T06:31:52,874 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(373): Skip log file (can't parse): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/oldWALs/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649097503 2023-08-10T06:31:53,283 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(146): Attempting to copy table info for:ns2:test-16916490353991 target: hdfs://localhost:43579/backupUT/backup_1691649108515/ns2/test-16916490353991 descriptor: 'ns2:test-16916490353991', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:31:53,283 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-08-10T06:31:53,285 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(151): Starting to write region info for table ns2:test-16916490353991 2023-08-10T06:31:53,690 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(158): Finished writing region info for table ns2:test-16916490353991 2023-08-10T06:31:54,101 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(146): Attempting to copy table info for:test-1691649035399 target: hdfs://localhost:43579/backupUT/backup_1691649108515/default/test-1691649035399 descriptor: 'test-1691649035399', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'f3', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-08-10T06:31:54,101 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(148): Finished copying tableinfo. 2023-08-10T06:31:54,103 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(151): Starting to write region info for table test-1691649035399 2023-08-10T06:31:54,509 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.BackupUtils(158): Finished writing region info for table test-1691649035399 2023-08-10T06:31:54,516 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.WALPlayer(360): add incremental job :hdfs://localhost:43579/backupUT/.tmp/backup_1691649108515 from hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543;hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649097536;hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 2023-08-10T06:31:54,522 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15918041, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:54,523 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-08-10T06:31:54,525 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:41464, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-08-10T06:31:54,527 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-08-10T06:31:54,527 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16916490353991 2023-08-10T06:31:54,530 DEBUG [hconnection-0x1f7295e2-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:54,531 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39314, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:54,532 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16916490353991: ns2:test-16916490353991; 2023-08-10T06:31:54,532 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1691649035399 2023-08-10T06:31:54,534 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1691649035399: test-1691649035399; 2023-08-10T06:31:54,534 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(694): Configuring 2 reduce partitions to match current region count for all tables 2023-08-10T06:31:54,534 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/hbase-staging/partitions_5796cbf9-0e86-4500-bba9-4876a6a30884 2023-08-10T06:31:54,946 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:54,946 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:54,947 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:54,947 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:56,012 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/hadoop-3828105704217526114.jar 2023-08-10T06:31:56,012 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:56,012 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:56,013 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:56,013 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:56,014 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:56,014 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:56,014 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:56,015 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:56,015 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:56,016 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:31:56,016 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-08-10T06:31:56,017 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-08-10T06:31:56,017 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-08-10T06:31:56,018 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-08-10T06:31:56,018 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-08-10T06:31:56,018 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-08-10T06:31:56,019 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-08-10T06:31:56,019 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-08-10T06:31:56,019 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-08-10T06:31:56,020 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-08-10T06:31:56,020 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-08-10T06:31:56,021 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:56,021 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:56,022 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.WALInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:56,022 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:56,022 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:56,023 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.MultiTableHFileOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-08-10T06:31:56,023 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-08-10T06:31:56,023 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16916490353991,default:test-1691649035399 2023-08-10T06:31:56,024 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:56,025 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-08-10T06:31:57,051 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.regionserver.wal.WALCellCodec, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/hadoop-1874901153503875460.jar 2023-08-10T06:31:57,072 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543 for WAL files 2023-08-10T06:31:57,073 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543 2023-08-10T06:31:57,074 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649097536 for WAL files 2023-08-10T06:31:57,074 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649097536 2023-08-10T06:31:57,074 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.WALInputFormat(355): Scanning hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 for WAL files 2023-08-10T06:31:57,075 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.WALInputFormat(381): Found hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 2023-08-10T06:31:57,151 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/mapreduce.cluster.local.dir/1691649117116/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-08-10T06:31:57,151 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/mapreduce.cluster.local.dir/1691649117116/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-08-10T06:31:57,205 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543 for hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649053543 (-9223372036854775808:9223372036854775807) length:18860 2023-08-10T06:31:57,210 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-08-10T06:31:57,210 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-08-10T06:31:57,235 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649097536 for hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0.1691649097536 (-9223372036854775808:9223372036854775807) length:3644 2023-08-10T06:31:57,238 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-08-10T06:31:57,238 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-08-10T06:31:57,260 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(196): Opening hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 for hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/WALs/jenkins-hbase3.apache.org,40209,1691649026642/jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1.1691649053506 (-9223372036854775808:9223372036854775807) length:1198 2023-08-10T06:31:57,263 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(244): Reached end of file. 2023-08-10T06:31:57,263 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.WALInputFormat$WALRecordReader(281): Closing reader 2023-08-10T06:31:57,283 DEBUG [pool-269-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df0a676, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:57,285 DEBUG [hconnection-0x3fa040e8-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:57,286 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39324, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:57,288 DEBUG [pool-269-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:57,289 DEBUG [pool-269-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-t20] 2023-08-10T06:31:57,289 DEBUG [pool-269-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-08-10T06:31:57,293 DEBUG [pool-269-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/backupUT/.tmp/backup_1691649108515/_temporary/0/_temporary/attempt_local812838052_0007_r_000000_0/ns2/test-16916490353991/f/60498e4c479c4582a61cc9a255c5ec17 is 38, key is row-t20/f:q1/1691649068577/Put/seqid=0 2023-08-10T06:31:57,295 WARN [Thread-1318 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:40209] Specified favored nodes: [jenkins-hbase3.apache.org:40209] 2023-08-10T06:31:57,721 DEBUG [pool-269-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cabd39a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:31:57,723 DEBUG [hconnection-0x6617514c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:31:57,725 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:39330, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:31:57,727 DEBUG [pool-269-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:31:57,731 DEBUG [pool-269-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row-test-1691649035399-1-0] 2023-08-10T06:31:57,731 DEBUG [pool-269-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-08-10T06:31:57,740 DEBUG [pool-269-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/backupUT/.tmp/backup_1691649108515/_temporary/0/_temporary/attempt_local812838052_0007_r_000001_0/default/test-1691649035399/f/6c3f10410a684c00babf1ec7a8917dd1 is 59, key is row-test-1691649035399-1-10/f:q1/1691649068469/Put/seqid=0 2023-08-10T06:31:57,741 WARN [Thread-1328 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:40209] Specified favored nodes: [jenkins-hbase3.apache.org:40209] 2023-08-10T06:31:58,185 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalTableBackupClient(325): Incremental copy HFiles is starting. dest=hdfs://localhost:43579/backupUT 2023-08-10T06:31:58,185 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalTableBackupClient(335): Setting incremental copy HFiles job name to : Incremental_Backup-HFileCopy-backup_1691649108515 2023-08-10T06:31:58,185 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob(379): Doing COPY_TYPE_DISTCP 2023-08-10T06:31:58,191 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob(388): DistCp options: [hdfs://localhost:43579/backupUT/.tmp/backup_1691649108515, hdfs://localhost:43579/backupUT] 2023-08-10T06:31:59,303 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(218): Progress: 100.0% subTask: 1.0 mapProgress: 1.0 2023-08-10T06:31:59,305 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob(138): Backup progress data "100%" has been updated to backup system table for backup_1691649108515 2023-08-10T06:31:59,305 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(226): Backup progress data updated to backup system table: "Progress: 100.0% - 16003 bytes copied." 2023-08-10T06:31:59,305 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(236): DistCp job-id: job_local441115963_0008 completed: true true 2023-08-10T06:31:59,308 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] mapreduce.MapReduceBackupCopyJob$BackupDistCp(239): Counters: 24 File System Counters FILE: Number of bytes read=5102044 FILE: Number of bytes written=9922982 FILE: Number of read operations=0 FILE: Number of large read operations=0 FILE: Number of write operations=0 HDFS: Number of bytes read=402653 HDFS: Number of bytes written=2483755 HDFS: Number of read operations=1363 HDFS: Number of large read operations=0 HDFS: Number of write operations=463 Map-Reduce Framework Map input records=10 Map output records=0 Input split bytes=342 Spilled Records=0 Failed Shuffles=0 Merged Map outputs=0 GC time elapsed (ms)=0 Total committed heap usage (bytes)=1549271040 File Input Format Counters Bytes Read=2444 File Output Format Counters Bytes Written=0 DistCp Counters Bytes Copied=16003 Bytes Expected=16003 Files Copied=3 DIR_COPY=7 2023-08-10T06:31:59,308 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.IncrementalTableBackupClient(346): Incremental copy HFiles from hdfs://localhost:43579/backupUT/.tmp/backup_1691649108515 to hdfs://localhost:43579/backupUT finished. 2023-08-10T06:31:59,319 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(433): deleted 0 original bulkload rows 2023-08-10T06:31:59,319 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(228): Unexpected Exception : Failed stage stage_4 in testing java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:160) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-08-10T06:31:59,320 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(240): BackupId=backup_1691649108515,startts=1691649112658,failedts=1691649119320,failedphase=INCREMENTAL_COPY,failedmessage=Failed stage stage_4 in testing 2023-08-10T06:31:59,320 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1518): Restoring backup:system from snapshot 2023-08-10T06:31:59,324 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster$12(2663): Client=jenkins//172.31.12.81 disable backup:system 2023-08-10T06:31:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=76, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-08-10T06:31:59,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=76 2023-08-10T06:31:59,329 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649119328"}]},"ts":"1691649119328"} 2023-08-10T06:31:59,330 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-08-10T06:31:59,331 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-08-10T06:31:59,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, UNASSIGN}] 2023-08-10T06:31:59,333 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=77, ppid=76, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, UNASSIGN 2023-08-10T06:31:59,334 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=77 updating hbase:meta row=4cb36c501bd471befb9ec81ac2da937f, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:31:59,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE; CloseRegionProcedure 4cb36c501bd471befb9ec81ac2da937f, server=jenkins-hbase3.apache.org,40209,1691649026642}] 2023-08-10T06:31:59,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=76 2023-08-10T06:31:59,487 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] handler.UnassignRegionHandler(119): Close 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:31:59,487 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1668): Closing 4cb36c501bd471befb9ec81ac2da937f, disabling compactions & flushes 2023-08-10T06:31:59,487 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1690): Closing region backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:59,487 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:59,487 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. after waiting 0 ms 2023-08-10T06:31:59,487 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:31:59,487 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(2819): Flushing 4cb36c501bd471befb9ec81ac2da937f 2/2 column families, dataSize=959 B heapSize=1.87 KB 2023-08-10T06:31:59,500 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/meta/bf4d6d7d7acc43868e99b97c828aa508 is 171, key is trslm:hdfs://localhost:43579/backupUT\x00ns2:test-16916490353991/meta:log-roll-map/1691649119312/Put/seqid=0 2023-08-10T06:31:59,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=76 2023-08-10T06:31:59,905 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=531 B at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/meta/bf4d6d7d7acc43868e99b97c828aa508 2023-08-10T06:31:59,911 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: bf4d6d7d7acc43868e99b97c828aa508 2023-08-10T06:31:59,914 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/00d8a73322564cffb15da7ce693af8e5 is 218, key is session:backup_1691649108515/session:context/1691649119304/Put/seqid=0 2023-08-10T06:31:59,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=76 2023-08-10T06:32:00,318 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=428 B at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/00d8a73322564cffb15da7ce693af8e5 2023-08-10T06:32:00,324 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 00d8a73322564cffb15da7ce693af8e5 2023-08-10T06:32:00,325 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/meta/bf4d6d7d7acc43868e99b97c828aa508 as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/bf4d6d7d7acc43868e99b97c828aa508 2023-08-10T06:32:00,331 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/bf4d6d7d7acc43868e99b97c828aa508, entries=4, sequenceid=73, filesize=5.7 K 2023-08-10T06:32:00,332 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/00d8a73322564cffb15da7ce693af8e5 as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/00d8a73322564cffb15da7ce693af8e5 2023-08-10T06:32:00,339 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/00d8a73322564cffb15da7ce693af8e5, entries=1, sequenceid=73, filesize=5.2 K 2023-08-10T06:32:00,340 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(3022): Finished flush of dataSize ~959 B/959, heapSize ~1.84 KB/1880, currentSize=0 B/0 for 4cb36c501bd471befb9ec81ac2da937f in 853ms, sequenceid=73, compaction requested=true 2023-08-10T06:32:00,343 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6cbd9d64f8534727b32d2198cce57056 2023-08-10T06:32:00,347 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: bf4d6d7d7acc43868e99b97c828aa508 2023-08-10T06:32:00,348 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e0b86ba5e37b4d82b985f61689fb9ccc 2023-08-10T06:32:00,348 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 255f97ba08f447aca89d284e51e6f514 2023-08-10T06:32:00,348 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a8091fba6ddb4ddd92b3536f0938b8be 2023-08-10T06:32:00,348 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 00d8a73322564cffb15da7ce693af8e5 2023-08-10T06:32:00,351 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/recovered.edits/76.seqid, newMaxSeqId=76, maxSeqId=59 2023-08-10T06:32:00,352 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:32:00,352 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1909): Closed backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:32:00,352 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] regionserver.HRegion(1622): Region close journal for 4cb36c501bd471befb9ec81ac2da937f: 2023-08-10T06:32:00,354 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=78}] handler.UnassignRegionHandler(163): Closed 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:32:00,354 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=77 updating hbase:meta row=4cb36c501bd471befb9ec81ac2da937f, regionState=CLOSED 2023-08-10T06:32:00,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=78, resume processing ppid=77 2023-08-10T06:32:00,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=78, ppid=77, state=SUCCESS; CloseRegionProcedure 4cb36c501bd471befb9ec81ac2da937f, server=jenkins-hbase3.apache.org,40209,1691649026642 in 1.0200 sec 2023-08-10T06:32:00,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=77, resume processing ppid=76 2023-08-10T06:32:00,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=77, ppid=76, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, UNASSIGN in 1.0250 sec 2023-08-10T06:32:00,358 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649120358"}]},"ts":"1691649120358"} 2023-08-10T06:32:00,359 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-08-10T06:32:00,361 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-08-10T06:32:00,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1414): Finished pid=76, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0370 sec 2023-08-10T06:32:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=76 2023-08-10T06:32:00,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1685): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1691649120441 table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:32:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1691649120442 (current time:1691649120442). 2023-08-10T06:32:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-08-10T06:32:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1691649120441 VERSION not specified, setting to 2 2023-08-10T06:32:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d78f615, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-08-10T06:32:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-08-10T06:32:00,450 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(459): Connection from 172.31.12.81:57780, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-08-10T06:32:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:32:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(770): No existing snapshot, attempting snapshot... 2023-08-10T06:32:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=79, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649120441 table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:32:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(1404): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649120441 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-08-10T06:32:00,454 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649120441 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-08-10T06:32:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-08-10T06:32:00,455 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649120441 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-08-10T06:32:00,457 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649120441 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-08-10T06:32:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-08-10T06:32:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-08-10T06:32:00,864 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649120441 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-08-10T06:32:00,865 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-08-10T06:32:00,865 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-08-10T06:32:00,865 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:32:00,866 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/bf4d6d7d7acc43868e99b97c828aa508] hfiles 2023-08-10T06:32:00,866 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056 2023-08-10T06:32:00,866 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/bf4d6d7d7acc43868e99b97c828aa508 2023-08-10T06:32:00,866 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:32:00,868 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/00d8a73322564cffb15da7ce693af8e5, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/255f97ba08f447aca89d284e51e6f514, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/a8091fba6ddb4ddd92b3536f0938b8be, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/e0b86ba5e37b4d82b985f61689fb9ccc] hfiles 2023-08-10T06:32:00,869 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/4): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/00d8a73322564cffb15da7ce693af8e5 2023-08-10T06:32:00,869 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/4): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/255f97ba08f447aca89d284e51e6f514 2023-08-10T06:32:00,869 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/4): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/a8091fba6ddb4ddd92b3536f0938b8be 2023-08-10T06:32:00,869 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (4/4): hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/e0b86ba5e37b4d82b985f61689fb9ccc 2023-08-10T06:32:00,874 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 4cb36c501bd471befb9ec81ac2da937f, NAME => 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-08-10T06:32:00,875 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649120441 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-08-10T06:32:00,876 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649120441 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-08-10T06:32:00,876 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1691649120441 2023-08-10T06:32:00,876 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1691649120441 2023-08-10T06:32:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-08-10T06:32:01,289 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649120441 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-08-10T06:32:01,297 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649120441 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-08-10T06:32:01,297 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1691649120441 to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1691649120441 2023-08-10T06:32:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-08-10T06:32:01,713 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649120441 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-08-10T06:32:01,713 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1410): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649120441 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-08-10T06:32:01,714 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1414): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1691649120441 table=backup:system type=FLUSH ttl=0 } in 1.2610 sec 2023-08-10T06:32:02,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=79 2023-08-10T06:32:02,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=80, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691649108539 type: FLUSH version: 2 ttl: 0 ) 2023-08-10T06:32:02,568 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(1015): Restore snapshot=snapshot_backup_system as table=backup:system 2023-08-10T06:32:02,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=80 2023-08-10T06:32:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=80 2023-08-10T06:32:02,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=80 2023-08-10T06:32:02,980 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-08-10T06:32:02,980 INFO [PEWorker-1 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/.tabledesc/.tableinfo.0000000005.635 2023-08-10T06:32:02,981 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-08-10T06:32:02,984 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691649108539 type: FLUSH version: 2 ttl: 0 2023-08-10T06:32:02,984 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system 2023-08-10T06:32:02,987 DEBUG [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-08-10T06:32:02,987 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:32:02,993 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/bf4d6d7d7acc43868e99b97c828aa508 to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/archive/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/bf4d6d7d7acc43868e99b97c828aa508 2023-08-10T06:32:02,993 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:32:02,996 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/00d8a73322564cffb15da7ce693af8e5 to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/archive/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/00d8a73322564cffb15da7ce693af8e5 2023-08-10T06:32:02,996 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:32:02,996 INFO [PEWorker-1 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691649108539 type: FLUSH version: 2 ttl: 0 2023-08-10T06:32:02,997 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1691649122997"}]},"ts":"9223372036854775807"} 2023-08-10T06:32:02,999 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-08-10T06:32:02,999 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 4cb36c501bd471befb9ec81ac2da937f, NAME => 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:32:02,999 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1691649122998"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1691649122998"}]},"ts":"1691649122998"} 2023-08-10T06:32:03,000 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-08-10T06:32:03,001 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-08-10T06:32:03,001 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 4cb36c501bd471befb9ec81ac2da937f, NAME => 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.', STARTKEY => '', ENDKEY => ''}] 2023-08-10T06:32:03,001 INFO [PEWorker-1 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-08-10T06:32:03,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=80, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1691649108539 type: FLUSH version: 2 ttl: 0 ) in 434 msec 2023-08-10T06:32:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=80 2023-08-10T06:32:03,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1691649120441" 2023-08-10T06:32:03,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(378): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1691649120441 2023-08-10T06:32:03,178 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.HMaster$11(2630): Client=jenkins//172.31.12.81 enable backup:system 2023-08-10T06:32:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] procedure2.ProcedureExecutor(1033): Stored pid=81, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-08-10T06:32:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-08-10T06:32:03,182 INFO [PEWorker-3 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-08-10T06:32:03,182 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649123182"}]},"ts":"1691649123182"} 2023-08-10T06:32:03,183 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-08-10T06:32:03,186 DEBUG [PEWorker-3 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-08-10T06:32:03,186 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, ASSIGN}] 2023-08-10T06:32:03,188 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, ASSIGN 2023-08-10T06:32:03,189 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-08-10T06:32:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-08-10T06:32:03,341 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=4cb36c501bd471befb9ec81ac2da937f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:32:03,342 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1685): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE; OpenRegionProcedure 4cb36c501bd471befb9ec81ac2da937f, server=jenkins-hbase3.apache.org,40209,1691649026642}] 2023-08-10T06:32:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-08-10T06:32:03,497 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(130): Open backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:32:03,497 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7254): Opening region: {ENCODED => 4cb36c501bd471befb9ec81ac2da937f, NAME => 'backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.', STARTKEY => '', ENDKEY => ''} 2023-08-10T06:32:03,497 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-08-10T06:32:03,498 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:32:03,498 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(888): Instantiated backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-08-10T06:32:03,498 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7296): checking encryption for 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:32:03,498 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7299): checking classloading for 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:32:03,499 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:32:03,500 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4cb36c501bd471befb9ec81ac2da937f columnFamilyName meta 2023-08-10T06:32:03,500 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:32:03,506 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/meta/6cbd9d64f8534727b32d2198cce57056 2023-08-10T06:32:03,506 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.HStore(324): Store=4cb36c501bd471befb9ec81ac2da937f/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:32:03,506 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:32:03,507 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4cb36c501bd471befb9ec81ac2da937f columnFamilyName session 2023-08-10T06:32:03,507 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-08-10T06:32:03,515 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/255f97ba08f447aca89d284e51e6f514 2023-08-10T06:32:03,521 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/a8091fba6ddb4ddd92b3536f0938b8be 2023-08-10T06:32:03,526 DEBUG [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/e0b86ba5e37b4d82b985f61689fb9ccc 2023-08-10T06:32:03,526 INFO [StoreOpener-4cb36c501bd471befb9ec81ac2da937f-1 {}] regionserver.HStore(324): Store=4cb36c501bd471befb9ec81ac2da937f/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-08-10T06:32:03,527 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:32:03,528 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:32:03,529 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-08-10T06:32:03,531 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1076): writing seq id for 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:32:03,531 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1093): Opened 4cb36c501bd471befb9ec81ac2da937f; next sequenceid=77; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9473281280, jitterRate=-0.11773192882537842}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-08-10T06:32:03,532 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(992): Region open journal for 4cb36c501bd471befb9ec81ac2da937f: 2023-08-10T06:32:03,533 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2552): Post open deploy tasks for backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f., pid=83, masterSystemTime=1691649123494 2023-08-10T06:32:03,535 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.CompactSplit(403): Add compact mark for store 4cb36c501bd471befb9ec81ac2da937f:session, priority=-2147483648, current under compaction store size is 1 2023-08-10T06:32:03,535 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-08-10T06:32:03,535 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-08-10T06:32:03,536 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 16834 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-08-10T06:32:03,536 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] regionserver.HStore(1534): 4cb36c501bd471befb9ec81ac2da937f/session is initiating minor compaction (all files) 2023-08-10T06:32:03,536 INFO [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 4cb36c501bd471befb9ec81ac2da937f/session in backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:32:03,536 INFO [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/e0b86ba5e37b4d82b985f61689fb9ccc, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/255f97ba08f447aca89d284e51e6f514, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/a8091fba6ddb4ddd92b3536f0938b8be] into tmpdir=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp, totalSize=16.4 K 2023-08-10T06:32:03,536 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2579): Finished post open deploy task for backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:32:03,536 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(158): Opened backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:32:03,537 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=82 updating hbase:meta row=4cb36c501bd471befb9ec81ac2da937f, regionState=OPEN, openSeqNum=77, regionLocation=jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:32:03,537 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] compactions.Compactor(223): Compacting e0b86ba5e37b4d82b985f61689fb9ccc, keycount=4, bloomtype=ROW, size=6.0 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1691649068420 2023-08-10T06:32:03,537 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] compactions.Compactor(223): Compacting 255f97ba08f447aca89d284e51e6f514, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1691649093284 2023-08-10T06:32:03,538 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] compactions.Compactor(223): Compacting a8091fba6ddb4ddd92b3536f0938b8be, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1691649108459 2023-08-10T06:32:03,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=83, resume processing ppid=82 2023-08-10T06:32:03,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1414): Finished pid=83, ppid=82, state=SUCCESS; OpenRegionProcedure 4cb36c501bd471befb9ec81ac2da937f, server=jenkins-hbase3.apache.org,40209,1691649026642 in 197 msec 2023-08-10T06:32:03,542 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1828): Finished subprocedure pid=82, resume processing ppid=81 2023-08-10T06:32:03,542 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1414): Finished pid=82, ppid=81, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=4cb36c501bd471befb9ec81ac2da937f, ASSIGN in 354 msec 2023-08-10T06:32:03,543 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1691649123543"}]},"ts":"1691649123543"} 2023-08-10T06:32:03,545 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-08-10T06:32:03,547 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-08-10T06:32:03,548 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1414): Finished pid=81, state=SUCCESS; EnableTableProcedure table=backup:system in 369 msec 2023-08-10T06:32:03,564 INFO [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4cb36c501bd471befb9ec81ac2da937f#session#compaction#17 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-08-10T06:32:03,565 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/7b5a8e64e5b14be89422fe1fb7149fee is 459, key is session:backup_1691649045860/session:context/1691649068420/Put/seqid=0 2023-08-10T06:32:03,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(1288): Checking to see if procedure is done pid=81 2023-08-10T06:32:03,784 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1525): Done restoring backup system table 2023-08-10T06:32:03,784 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1552): Deleting snapshot_backup_system from the system 2023-08-10T06:32:03,796 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] master.MasterRpcServices(759): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-08-10T06:32:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46449 {}] snapshot.SnapshotManager(378): Deleting snapshot: snapshot_backup_system 2023-08-10T06:32:03,800 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(1557): Done deleting backup system table snapshot 2023-08-10T06:32:03,800 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: INCREMENTAL_COPY 2023-08-10T06:32:03,801 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:43579/backupUT/backup_1691649108515/ns2/test-16916490353991 done. 2023-08-10T06:32:03,802 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(208): hdfs://localhost:43579/backupUT/backup_1691649108515/ns2 is empty, remove it. 2023-08-10T06:32:03,802 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:43579/backupUT/backup_1691649108515/default/test-1691649035399 done. 2023-08-10T06:32:03,803 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(208): hdfs://localhost:43579/backupUT/backup_1691649108515/default is empty, remove it. 2023-08-10T06:32:03,805 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] impl.BackupSystemTable(637): Finish backup exclusive operation 2023-08-10T06:32:03,810 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] impl.TableBackupClient(246): Backup backup_1691649108515 failed. Backup session finished. Status: FAILURE 2023-08-10T06:32:03,810 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:32:03,810 ERROR [Listener at jenkins-hbase3.apache.org/40387 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:168) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.runBackupAndFailAtStage(TestIncrementalBackupWithFailures.java:149) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.incrementalBackupWithFailures(TestIncrementalBackupWithFailures.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestIncrementalBackupWithFailures.testIncBackupRestore(TestIncrementalBackupWithFailures.java:123) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.run(ParentRunner.java:413) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:128) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.Suite.runChild(Suite.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$IncrementalTableBackupClientForTest.execute(TestBackupBase.java:160) ~[test-classes/:?] ... 41 more 2023-08-10T06:32:03,815 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:32:03,835 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] hbase.ResourceChecker(175): after: backup.TestIncrementalBackupWithFailures#testIncBackupRestore[0] Thread=601 (was 586) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_810807640_16 at /127.0.0.1:51966 [Receiving block BP-1419340424-172.31.12.81-1691649020475:blk_1073741958_1134] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_810807640_16 at /127.0.0.1:51976 [Receiving block BP-1419340424-172.31.12.81-1691649020475:blk_1073741959_1135] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS:0;jenkins-hbase3:40209-shortCompactions-0 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:940) org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:880) org.apache.hadoop.hdfs.DFSOutputStream.closeImpl(DFSOutputStream.java:863) org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:818) org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72) org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.finishClose(HFileWriterImpl.java:858) org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.close(HFileWriterImpl.java:679) org.apache.hadoop.hbase.regionserver.StoreFileWriter.close(StoreFileWriter.java:377) org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:72) org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:40) org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Compactor.java:396) org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:64) org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:122) org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1176) org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:2341) org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.doCompaction(CompactSplit.java:667) org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.run(CompactSplit.java:716) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_4843596_16 at /127.0.0.1:34380 [Waiting for operation #7] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_4843596_16 at /127.0.0.1:52028 [Waiting for operation #25] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_4843596_16 at /127.0.0.1:34320 [Waiting for operation #9] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1419340424-172.31.12.81-1691649020475:blk_1073741958_1134, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_4843596_16 at /127.0.0.1:34292 [Waiting for operation #9] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,40209,1691649026642' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x3e2a44a9-shared-pool-33 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1df1c5f3-shared-pool-25 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:55224@0x4f49edb1-SendThread(127.0.0.1:55224) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:228) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:55224@0x4f49edb1 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$490/974594068.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1df1c5f3-shared-pool-24 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/cluster_336bfe0c-247d-9555-0740-69ce104bc2cc/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1419340424-172.31.12.81-1691649020475:blk_1073741960_1136, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/7b5a8e64e5b14be89422fe1fb7149fee java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:678) Potentially hanging thread: (jenkins-hbase3.apache.org,46449,1691649023646)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_810807640_16 at /127.0.0.1:51990 [Receiving block BP-1419340424-172.31.12.81-1691649020475:blk_1073741960_1136] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_4843596_16 at /127.0.0.1:34366 [Waiting for operation #3] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1419340424-172.31.12.81-1691649020475:blk_1073741959_1135, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/cluster_336bfe0c-247d-9555-0740-69ce104bc2cc/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:55224@0x4f49edb1-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: rs(jenkins-hbase3.apache.org,40209,1691649026642)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=685 (was 667) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=81 (was 149), ProcessCount=168 (was 171), AvailableMemoryMB=2233 (was 1793) - AvailableMemoryMB LEAK? - 2023-08-10T06:32:03,835 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] hbase.ResourceChecker(130): Thread=601 is superior to 500 2023-08-10T06:32:03,843 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-08-10T06:32:03,843 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:32:03,843 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-08-10T06:32:03,843 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] util.JVMClusterUtil(257): Found active master hash=2052150187, stopped=false 2023-08-10T06:32:03,844 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,46449,1691649023646 2023-08-10T06:32:03,847 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] procedure2.ProcedureExecutor(631): Stopping 2023-08-10T06:32:03,848 DEBUG [Listener at jenkins-hbase3.apache.org/40387 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:32:03,848 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,40209,1691649026642' ***** 2023-08-10T06:32:03,848 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] regionserver.HRegionServer(2527): STOPPED: Shutdown requested 2023-08-10T06:32:03,848 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-08-10T06:32:03,861 INFO [RS:0;jenkins-hbase3:40209 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@31f20006{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-08-10T06:32:03,865 INFO [RS:0;jenkins-hbase3:40209 {}] server.AbstractConnector(383): Stopped ServerConnector@461e973d{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-08-10T06:32:03,865 INFO [RS:0;jenkins-hbase3:40209 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-08-10T06:32:03,866 INFO [RS:0;jenkins-hbase3:40209 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@3e6c1de8{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-08-10T06:32:03,866 INFO [RS:0;jenkins-hbase3:40209 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5fac4aa4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/hadoop.log.dir/,STOPPED} 2023-08-10T06:32:03,869 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HeapMemoryManager(220): Stopping 2023-08-10T06:32:03,869 INFO [RS:0;jenkins-hbase3:40209 {}] flush.RegionServerFlushTableProcedureManager(117): Stopping region server flush procedure manager gracefully. 2023-08-10T06:32:03,869 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-08-10T06:32:03,869 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-08-10T06:32:03,869 INFO [RS:0;jenkins-hbase3:40209 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-08-10T06:32:03,869 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(3524): Received CLOSE for faa733e2577a03b5cd26e4424a6f4087 2023-08-10T06:32:03,870 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(3524): Received CLOSE for e4d21e17fa5b460ebe5cf90f2eb64bca 2023-08-10T06:32:03,870 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(3524): Received CLOSE for 66e77e1fb4a2e8bff5d5df89490b6752 2023-08-10T06:32:03,870 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(3524): Received CLOSE for 41ac6ed181c2526bbcdb3ef2c04e3054 2023-08-10T06:32:03,870 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(3524): Received CLOSE for ab0a54958c666e251775ab33a9efc343 2023-08-10T06:32:03,870 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(3524): Received CLOSE for 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:32:03,870 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(3524): Received CLOSE for 22e2ac566be23eb7afc4a9349ef3e613 2023-08-10T06:32:03,870 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,40209,1691649026642 2023-08-10T06:32:03,871 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing faa733e2577a03b5cd26e4424a6f4087, disabling compactions & flushes 2023-08-10T06:32:03,871 DEBUG [RS:0;jenkins-hbase3:40209 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:32:03,871 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1691649029519.faa733e2577a03b5cd26e4424a6f4087. 2023-08-10T06:32:03,871 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1691649029519.faa733e2577a03b5cd26e4424a6f4087. 2023-08-10T06:32:03,871 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1691649029519.faa733e2577a03b5cd26e4424a6f4087. after waiting 0 ms 2023-08-10T06:32:03,871 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-08-10T06:32:03,871 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1691649029519.faa733e2577a03b5cd26e4424a6f4087. 2023-08-10T06:32:03,871 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-08-10T06:32:03,871 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing faa733e2577a03b5cd26e4424a6f4087 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-08-10T06:32:03,875 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-08-10T06:32:03,885 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/namespace/faa733e2577a03b5cd26e4424a6f4087/.tmp/info/7673b6ff2f464f4cbd875a7fb74dd85d is 45, key is default/info:d/1691649030902/Put/seqid=0 2023-08-10T06:32:03,894 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/namespace/faa733e2577a03b5cd26e4424a6f4087/.tmp/info/7673b6ff2f464f4cbd875a7fb74dd85d 2023-08-10T06:32:03,905 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 7673b6ff2f464f4cbd875a7fb74dd85d 2023-08-10T06:32:03,906 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/namespace/faa733e2577a03b5cd26e4424a6f4087/.tmp/info/7673b6ff2f464f4cbd875a7fb74dd85d as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/namespace/faa733e2577a03b5cd26e4424a6f4087/info/7673b6ff2f464f4cbd875a7fb74dd85d 2023-08-10T06:32:03,912 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/namespace/faa733e2577a03b5cd26e4424a6f4087/info/7673b6ff2f464f4cbd875a7fb74dd85d, entries=7, sequenceid=11, filesize=5.1 K 2023-08-10T06:32:03,913 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for faa733e2577a03b5cd26e4424a6f4087 in 42ms, sequenceid=11, compaction requested=false 2023-08-10T06:32:03,916 DEBUG [StoreFileCloser-faa733e2577a03b5cd26e4424a6f4087-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 7673b6ff2f464f4cbd875a7fb74dd85d 2023-08-10T06:32:03,921 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/namespace/faa733e2577a03b5cd26e4424a6f4087/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-08-10T06:32:03,922 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:32:03,923 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1691649029519.faa733e2577a03b5cd26e4424a6f4087. 2023-08-10T06:32:03,923 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for faa733e2577a03b5cd26e4424a6f4087: 2023-08-10T06:32:03,923 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1691649029519.faa733e2577a03b5cd26e4424a6f4087. 2023-08-10T06:32:03,924 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing e4d21e17fa5b460ebe5cf90f2eb64bca, disabling compactions & flushes 2023-08-10T06:32:03,924 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1691649047137.e4d21e17fa5b460ebe5cf90f2eb64bca. 2023-08-10T06:32:03,924 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1691649047137.e4d21e17fa5b460ebe5cf90f2eb64bca. 2023-08-10T06:32:03,924 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1691649047137.e4d21e17fa5b460ebe5cf90f2eb64bca. after waiting 0 ms 2023-08-10T06:32:03,924 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1691649047137.e4d21e17fa5b460ebe5cf90f2eb64bca. 2023-08-10T06:32:03,927 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system_bulk/e4d21e17fa5b460ebe5cf90f2eb64bca/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-08-10T06:32:03,928 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:32:03,929 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1691649047137.e4d21e17fa5b460ebe5cf90f2eb64bca. 2023-08-10T06:32:03,929 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for e4d21e17fa5b460ebe5cf90f2eb64bca: 2023-08-10T06:32:03,929 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1691649047137.e4d21e17fa5b460ebe5cf90f2eb64bca. 2023-08-10T06:32:03,929 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 66e77e1fb4a2e8bff5d5df89490b6752, disabling compactions & flushes 2023-08-10T06:32:03,930 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16916490353992,,1691649040330.66e77e1fb4a2e8bff5d5df89490b6752. 2023-08-10T06:32:03,930 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16916490353992,,1691649040330.66e77e1fb4a2e8bff5d5df89490b6752. 2023-08-10T06:32:03,930 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16916490353992,,1691649040330.66e77e1fb4a2e8bff5d5df89490b6752. after waiting 0 ms 2023-08-10T06:32:03,930 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16916490353992,,1691649040330.66e77e1fb4a2e8bff5d5df89490b6752. 2023-08-10T06:32:03,933 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns3/test-16916490353992/66e77e1fb4a2e8bff5d5df89490b6752/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-08-10T06:32:03,933 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:32:03,934 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16916490353992,,1691649040330.66e77e1fb4a2e8bff5d5df89490b6752. 2023-08-10T06:32:03,934 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 66e77e1fb4a2e8bff5d5df89490b6752: 2023-08-10T06:32:03,934 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16916490353992,,1691649040330.66e77e1fb4a2e8bff5d5df89490b6752. 2023-08-10T06:32:03,934 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 41ac6ed181c2526bbcdb3ef2c04e3054, disabling compactions & flushes 2023-08-10T06:32:03,934 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:32:03,934 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:32:03,934 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. after waiting 0 ms 2023-08-10T06:32:03,934 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:32:03,934 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 41ac6ed181c2526bbcdb3ef2c04e3054 2/2 column families, dataSize=5.30 KB heapSize=12.88 KB 2023-08-10T06:32:03,947 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/.tmp/f/92d75f3ac76b4d73b28bab683b54591d is 59, key is row-test-1691649035399-1-10/f:q1/1691649068469/Put/seqid=0 2023-08-10T06:32:03,977 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 7b5a8e64e5b14be89422fe1fb7149fee 2023-08-10T06:32:03,978 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/7b5a8e64e5b14be89422fe1fb7149fee as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/7b5a8e64e5b14be89422fe1fb7149fee 2023-08-10T06:32:03,984 INFO [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 4cb36c501bd471befb9ec81ac2da937f/session of 4cb36c501bd471befb9ec81ac2da937f into 7b5a8e64e5b14be89422fe1fb7149fee(size=6.5 K), total size for store is 6.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-08-10T06:32:03,984 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 4cb36c501bd471befb9ec81ac2da937f: 2023-08-10T06:32:03,984 INFO [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f., storeName=4cb36c501bd471befb9ec81ac2da937f/session, priority=13, startTime=1691649123533; duration=0sec 2023-08-10T06:32:03,984 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-08-10T06:32:03,984 DEBUG [RS:0;jenkins-hbase3:40209-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4cb36c501bd471befb9ec81ac2da937f:session 2023-08-10T06:32:03,985 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-08-10T06:32:03,985 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(3524): Received CLOSE for 1588230740 2023-08-10T06:32:03,985 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1579): Waiting on 5 regions to close 2023-08-10T06:32:03,985 DEBUG [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1583): Online Regions={41ac6ed181c2526bbcdb3ef2c04e3054=test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054., 1588230740=hbase:meta,,1.1588230740, ab0a54958c666e251775ab33a9efc343=ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343., 4cb36c501bd471befb9ec81ac2da937f=backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f., 22e2ac566be23eb7afc4a9349ef3e613=ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613.} 2023-08-10T06:32:03,986 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-08-10T06:32:03,986 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-08-10T06:32:03,986 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-08-10T06:32:03,986 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-08-10T06:32:03,986 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-08-10T06:32:03,987 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=18.81 KB heapSize=32.60 KB 2023-08-10T06:32:03,990 DEBUG [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 22e2ac566be23eb7afc4a9349ef3e613, 41ac6ed181c2526bbcdb3ef2c04e3054, 4cb36c501bd471befb9ec81ac2da937f, ab0a54958c666e251775ab33a9efc343 2023-08-10T06:32:04,013 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/meta/1588230740/.tmp/info/f3ec84b37f78442db0c31f3590a8bdf6 is 159, key is ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343./info:regioninfo/1691649039323/Put/seqid=0 2023-08-10T06:32:04,191 DEBUG [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 22e2ac566be23eb7afc4a9349ef3e613, 41ac6ed181c2526bbcdb3ef2c04e3054, 4cb36c501bd471befb9ec81ac2da937f, ab0a54958c666e251775ab33a9efc343 2023-08-10T06:32:04,357 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.30 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/.tmp/f/92d75f3ac76b4d73b28bab683b54591d 2023-08-10T06:32:04,362 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 92d75f3ac76b4d73b28bab683b54591d 2023-08-10T06:32:04,363 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/.tmp/f/92d75f3ac76b4d73b28bab683b54591d as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/f/92d75f3ac76b4d73b28bab683b54591d 2023-08-10T06:32:04,367 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/f/92d75f3ac76b4d73b28bab683b54591d, entries=99, sequenceid=217, filesize=10.5 K 2023-08-10T06:32:04,368 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~5.30 KB/5425, heapSize ~12.61 KB/12912, currentSize=0 B/0 for 41ac6ed181c2526bbcdb3ef2c04e3054 in 434ms, sequenceid=217, compaction requested=false 2023-08-10T06:32:04,369 DEBUG [StoreFileCloser-41ac6ed181c2526bbcdb3ef2c04e3054-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: fa1d8c8d9a7f448881583cf4837e9f5e 2023-08-10T06:32:04,370 DEBUG [StoreFileCloser-41ac6ed181c2526bbcdb3ef2c04e3054-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 92d75f3ac76b4d73b28bab683b54591d 2023-08-10T06:32:04,371 DEBUG [StoreFileCloser-41ac6ed181c2526bbcdb3ef2c04e3054-f3-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 469af5dd68d44838a76a85d58d415d3a 2023-08-10T06:32:04,374 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/default/test-1691649035399/41ac6ed181c2526bbcdb3ef2c04e3054/recovered.edits/220.seqid, newMaxSeqId=220, maxSeqId=106 2023-08-10T06:32:04,375 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:32:04,376 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:32:04,376 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 41ac6ed181c2526bbcdb3ef2c04e3054: 2023-08-10T06:32:04,376 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1691649035399,,1691649035884.41ac6ed181c2526bbcdb3ef2c04e3054. 2023-08-10T06:32:04,376 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing ab0a54958c666e251775ab33a9efc343, disabling compactions & flushes 2023-08-10T06:32:04,376 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343. 2023-08-10T06:32:04,376 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343. 2023-08-10T06:32:04,376 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343. after waiting 0 ms 2023-08-10T06:32:04,376 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343. 2023-08-10T06:32:04,376 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing ab0a54958c666e251775ab33a9efc343 1/1 column families, dataSize=170 B heapSize=816 B 2023-08-10T06:32:04,379 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns2/test-16916490353991/ab0a54958c666e251775ab33a9efc343/.tmp/f/e958e6f3b3e445868e0802bb2518e86a is 38, key is row-t20/f:q1/1691649068577/Put/seqid=0 2023-08-10T06:32:04,391 DEBUG [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 22e2ac566be23eb7afc4a9349ef3e613, 4cb36c501bd471befb9ec81ac2da937f, ab0a54958c666e251775ab33a9efc343 2023-08-10T06:32:04,418 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.41 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/meta/1588230740/.tmp/info/f3ec84b37f78442db0c31f3590a8bdf6 2023-08-10T06:32:04,424 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f3ec84b37f78442db0c31f3590a8bdf6 2023-08-10T06:32:04,436 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/meta/1588230740/.tmp/table/3e5faf7eec9149c293ae437b484ad949 is 59, key is ns2:test-16916490353991/table:state/1691649039332/Put/seqid=0 2023-08-10T06:32:04,591 DEBUG [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 22e2ac566be23eb7afc4a9349ef3e613, 4cb36c501bd471befb9ec81ac2da937f, ab0a54958c666e251775ab33a9efc343 2023-08-10T06:32:04,620 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-08-10T06:32:04,620 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-08-10T06:32:04,786 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns2/test-16916490353991/ab0a54958c666e251775ab33a9efc343/.tmp/f/e958e6f3b3e445868e0802bb2518e86a 2023-08-10T06:32:04,791 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e958e6f3b3e445868e0802bb2518e86a 2023-08-10T06:32:04,792 DEBUG [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 22e2ac566be23eb7afc4a9349ef3e613, 4cb36c501bd471befb9ec81ac2da937f, ab0a54958c666e251775ab33a9efc343 2023-08-10T06:32:04,792 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns2/test-16916490353991/ab0a54958c666e251775ab33a9efc343/.tmp/f/e958e6f3b3e445868e0802bb2518e86a as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns2/test-16916490353991/ab0a54958c666e251775ab33a9efc343/f/e958e6f3b3e445868e0802bb2518e86a 2023-08-10T06:32:04,797 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns2/test-16916490353991/ab0a54958c666e251775ab33a9efc343/f/e958e6f3b3e445868e0802bb2518e86a, entries=5, sequenceid=111, filesize=5.0 K 2023-08-10T06:32:04,797 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~170 B/170, heapSize ~800 B/800, currentSize=0 B/0 for ab0a54958c666e251775ab33a9efc343 in 421ms, sequenceid=111, compaction requested=false 2023-08-10T06:32:04,798 DEBUG [StoreFileCloser-ab0a54958c666e251775ab33a9efc343-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9df456b3ebab4661981987c1a670948b 2023-08-10T06:32:04,799 DEBUG [StoreFileCloser-ab0a54958c666e251775ab33a9efc343-f-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e958e6f3b3e445868e0802bb2518e86a 2023-08-10T06:32:04,804 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns2/test-16916490353991/ab0a54958c666e251775ab33a9efc343/recovered.edits/114.seqid, newMaxSeqId=114, maxSeqId=1 2023-08-10T06:32:04,804 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:32:04,805 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343. 2023-08-10T06:32:04,805 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for ab0a54958c666e251775ab33a9efc343: 2023-08-10T06:32:04,805 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16916490353991,,1691649038127.ab0a54958c666e251775ab33a9efc343. 2023-08-10T06:32:04,805 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 4cb36c501bd471befb9ec81ac2da937f, disabling compactions & flushes 2023-08-10T06:32:04,805 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:32:04,805 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:32:04,805 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. after waiting 0 ms 2023-08-10T06:32:04,805 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:32:04,805 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 4cb36c501bd471befb9ec81ac2da937f 2/2 column families, dataSize=296 B heapSize=960 B 2023-08-10T06:32:04,808 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/030db8debc2145c590d0e7ae84633584 is 256, key is session:backup_1691649108515/session:context/1691649123804/Put/seqid=0 2023-08-10T06:32:04,840 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.40 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/meta/1588230740/.tmp/table/3e5faf7eec9149c293ae437b484ad949 2023-08-10T06:32:04,845 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3e5faf7eec9149c293ae437b484ad949 2023-08-10T06:32:04,845 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/meta/1588230740/.tmp/info/f3ec84b37f78442db0c31f3590a8bdf6 as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/meta/1588230740/info/f3ec84b37f78442db0c31f3590a8bdf6 2023-08-10T06:32:04,850 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/meta/1588230740/info/f3ec84b37f78442db0c31f3590a8bdf6, entries=79, sequenceid=83, filesize=15.1 K 2023-08-10T06:32:04,850 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/meta/1588230740/.tmp/table/3e5faf7eec9149c293ae437b484ad949 as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/meta/1588230740/table/3e5faf7eec9149c293ae437b484ad949 2023-08-10T06:32:04,855 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/meta/1588230740/table/3e5faf7eec9149c293ae437b484ad949, entries=15, sequenceid=83, filesize=5.8 K 2023-08-10T06:32:04,856 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~18.81 KB/19263, heapSize ~32.32 KB/33096, currentSize=0 B/0 for 1588230740 in 870ms, sequenceid=83, compaction requested=false 2023-08-10T06:32:04,858 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: f3ec84b37f78442db0c31f3590a8bdf6 2023-08-10T06:32:04,860 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 3e5faf7eec9149c293ae437b484ad949 2023-08-10T06:32:04,864 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/hbase/meta/1588230740/recovered.edits/86.seqid, newMaxSeqId=86, maxSeqId=1 2023-08-10T06:32:04,864 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:32:04,864 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-08-10T06:32:04,865 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-08-10T06:32:04,865 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-08-10T06:32:04,865 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-08-10T06:32:04,992 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1579): Waiting on 2 regions to close 2023-08-10T06:32:04,992 DEBUG [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1583): Online Regions={4cb36c501bd471befb9ec81ac2da937f=backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f., 22e2ac566be23eb7afc4a9349ef3e613=ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613.} 2023-08-10T06:32:04,992 DEBUG [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1609): Waiting on 22e2ac566be23eb7afc4a9349ef3e613, 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:32:05,192 DEBUG [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1609): Waiting on 22e2ac566be23eb7afc4a9349ef3e613, 4cb36c501bd471befb9ec81ac2da937f 2023-08-10T06:32:05,212 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=296 B at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/030db8debc2145c590d0e7ae84633584 2023-08-10T06:32:05,218 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 030db8debc2145c590d0e7ae84633584 2023-08-10T06:32:05,219 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/.tmp/session/030db8debc2145c590d0e7ae84633584 as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/030db8debc2145c590d0e7ae84633584 2023-08-10T06:32:05,224 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/030db8debc2145c590d0e7ae84633584, entries=2, sequenceid=82, filesize=5.2 K 2023-08-10T06:32:05,225 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~296 B/296, heapSize ~688 B/688, currentSize=0 B/0 for 4cb36c501bd471befb9ec81ac2da937f in 420ms, sequenceid=82, compaction requested=false 2023-08-10T06:32:05,227 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-meta-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 6cbd9d64f8534727b32d2198cce57056 2023-08-10T06:32:05,229 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e0b86ba5e37b4d82b985f61689fb9ccc 2023-08-10T06:32:05,231 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 255f97ba08f447aca89d284e51e6f514 2023-08-10T06:32:05,232 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a8091fba6ddb4ddd92b3536f0938b8be 2023-08-10T06:32:05,233 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/e0b86ba5e37b4d82b985f61689fb9ccc, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/255f97ba08f447aca89d284e51e6f514, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/a8091fba6ddb4ddd92b3536f0938b8be] to archive 2023-08-10T06:32:05,234 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-08-10T06:32:05,235 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: e0b86ba5e37b4d82b985f61689fb9ccc 2023-08-10T06:32:05,236 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/e0b86ba5e37b4d82b985f61689fb9ccc to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/archive/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/e0b86ba5e37b4d82b985f61689fb9ccc 2023-08-10T06:32:05,237 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 255f97ba08f447aca89d284e51e6f514 2023-08-10T06:32:05,239 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/255f97ba08f447aca89d284e51e6f514 to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/archive/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/255f97ba08f447aca89d284e51e6f514 2023-08-10T06:32:05,239 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: a8091fba6ddb4ddd92b3536f0938b8be 2023-08-10T06:32:05,240 DEBUG [StoreCloser-backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/a8091fba6ddb4ddd92b3536f0938b8be to hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/archive/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/session/a8091fba6ddb4ddd92b3536f0938b8be 2023-08-10T06:32:05,242 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 7b5a8e64e5b14be89422fe1fb7149fee 2023-08-10T06:32:05,243 DEBUG [StoreFileCloser-4cb36c501bd471befb9ec81ac2da937f-session-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 030db8debc2145c590d0e7ae84633584 2023-08-10T06:32:05,248 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/backup/system/4cb36c501bd471befb9ec81ac2da937f/recovered.edits/85.seqid, newMaxSeqId=85, maxSeqId=76 2023-08-10T06:32:05,248 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:32:05,249 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:32:05,249 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 4cb36c501bd471befb9ec81ac2da937f: 2023-08-10T06:32:05,249 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1691649046010.4cb36c501bd471befb9ec81ac2da937f. 2023-08-10T06:32:05,249 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 22e2ac566be23eb7afc4a9349ef3e613, disabling compactions & flushes 2023-08-10T06:32:05,249 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613. 2023-08-10T06:32:05,249 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613. 2023-08-10T06:32:05,249 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613. after waiting 0 ms 2023-08-10T06:32:05,249 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613. 2023-08-10T06:32:05,252 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/data/ns4/test-16916490353993/22e2ac566be23eb7afc4a9349ef3e613/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-08-10T06:32:05,253 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-08-10T06:32:05,253 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613. 2023-08-10T06:32:05,253 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 22e2ac566be23eb7afc4a9349ef3e613: 2023-08-10T06:32:05,254 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16916490353993,,1691649042454.22e2ac566be23eb7afc4a9349ef3e613. 2023-08-10T06:32:05,392 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,40209,1691649026642; all regions closed. 2023-08-10T06:32:05,409 DEBUG [RS:0;jenkins-hbase3:40209 {}] wal.AbstractFSWAL(1062): Moved 5 WAL file(s) to /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/oldWALs 2023-08-10T06:32:05,409 INFO [RS:0;jenkins-hbase3:40209 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C40209%2C1691649026642.meta:.meta(num 1691649112695) 2023-08-10T06:32:05,416 DEBUG [RS:0;jenkins-hbase3:40209 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/oldWALs 2023-08-10T06:32:05,416 INFO [RS:0;jenkins-hbase3:40209 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-0:(num 1691649112711) 2023-08-10T06:32:05,425 DEBUG [RS:0;jenkins-hbase3:40209 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/oldWALs 2023-08-10T06:32:05,425 INFO [RS:0;jenkins-hbase3:40209 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C40209%2C1691649026642.jenkins-hbase3.apache.org%2C40209%2C1691649026642.regiongroup-1:(num 1691649112676) 2023-08-10T06:32:05,425 DEBUG [RS:0;jenkins-hbase3:40209 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:32:05,425 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.LeaseManager(133): Closed leases 2023-08-10T06:32:05,425 INFO [RS:0;jenkins-hbase3:40209 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-08-10T06:32:05,426 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-08-10T06:32:05,428 INFO [RS:0;jenkins-hbase3:40209 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:40209 2023-08-10T06:32:05,439 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,40209,1691649026642] 2023-08-10T06:32:05,440 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,40209,1691649026642; numProcessing=1 2023-08-10T06:32:05,441 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,40209,1691649026642 expired; onlineServers=0 2023-08-10T06:32:05,441 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2513): ***** STOPPING region server 'jenkins-hbase3.apache.org,46449,1691649023646' ***** 2023-08-10T06:32:05,441 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2527): STOPPED: Cluster shutdown set; onlineServer=0 2023-08-10T06:32:05,442 DEBUG [M:0;jenkins-hbase3:46449 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@509cf78b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-08-10T06:32:05,442 INFO [M:0;jenkins-hbase3:46449 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-08-10T06:32:05,499 INFO [M:0;jenkins-hbase3:46449 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@233c5f70{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-08-10T06:32:05,499 INFO [M:0;jenkins-hbase3:46449 {}] server.AbstractConnector(383): Stopped ServerConnector@af7902f{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-08-10T06:32:05,500 INFO [M:0;jenkins-hbase3:46449 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-08-10T06:32:05,500 INFO [M:0;jenkins-hbase3:46449 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@496f5e72{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-08-10T06:32:05,501 INFO [M:0;jenkins-hbase3:46449 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@1591e208{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/hadoop.log.dir/,STOPPED} 2023-08-10T06:32:05,501 INFO [M:0;jenkins-hbase3:46449 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,46449,1691649023646 2023-08-10T06:32:05,501 INFO [M:0;jenkins-hbase3:46449 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,46449,1691649023646; all regions closed. 2023-08-10T06:32:05,501 DEBUG [M:0;jenkins-hbase3:46449 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:32:05,501 INFO [M:0;jenkins-hbase3:46449 {}] master.HMaster(1664): Stopping master jetty server 2023-08-10T06:32:05,502 INFO [M:0;jenkins-hbase3:46449 {}] server.AbstractConnector(383): Stopped ServerConnector@528f8ba6{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-08-10T06:32:05,502 DEBUG [M:0;jenkins-hbase3:46449 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-08-10T06:32:05,503 INFO [M:0;jenkins-hbase3:46449 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-08-10T06:32:05,503 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-08-10T06:32:05,503 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1691649028406 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1691649028406,5,FailOnTimeoutGroup] 2023-08-10T06:32:05,503 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1691649028402 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1691649028402,5,FailOnTimeoutGroup] 2023-08-10T06:32:05,503 DEBUG [M:0;jenkins-hbase3:46449 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-08-10T06:32:05,503 DEBUG [M:0;jenkins-hbase3:46449 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-08-10T06:32:05,503 INFO [M:0;jenkins-hbase3:46449 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-08-10T06:32:05,503 DEBUG [M:0;jenkins-hbase3:46449 {}] master.HMaster(1687): Stopping service threads 2023-08-10T06:32:05,503 INFO [M:0;jenkins-hbase3:46449 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-08-10T06:32:05,504 ERROR [M:0;jenkins-hbase3:46449 {}] procedure2.ProcedureExecutor(655): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-08-10T06:32:05,504 INFO [M:0;jenkins-hbase3:46449 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-08-10T06:32:05,504 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-08-10T06:32:05,505 WARN [M:0;jenkins-hbase3:46449 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-08-10T06:32:05,505 INFO [M:0;jenkins-hbase3:46449 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-08-10T06:32:05,505 INFO [M:0;jenkins-hbase3:46449 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-08-10T06:32:05,506 DEBUG [M:0;jenkins-hbase3:46449 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-08-10T06:32:05,518 INFO [M:0;jenkins-hbase3:46449 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:32:05,518 DEBUG [M:0;jenkins-hbase3:46449 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:32:05,518 DEBUG [M:0;jenkins-hbase3:46449 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-08-10T06:32:05,518 DEBUG [M:0;jenkins-hbase3:46449 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:32:05,518 INFO [M:0;jenkins-hbase3:46449 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=326.08 KB heapSize=390.06 KB 2023-08-10T06:32:05,530 DEBUG [M:0;jenkins-hbase3:46449 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/145321bb47ef4b5483ee4a4b5a9e6535 is 88, key is hbase:meta,,1/info:sn/1691649028868/Put/seqid=0 2023-08-10T06:32:05,540 INFO [RS:0;jenkins-hbase3:40209 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,40209,1691649026642; zookeeper connection closed. 2023-08-10T06:32:05,540 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7bedb501 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7bedb501 2023-08-10T06:32:05,540 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-08-10T06:32:05,900 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-08-10T06:32:05,936 INFO [M:0;jenkins-hbase3:46449 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/145321bb47ef4b5483ee4a4b5a9e6535 2023-08-10T06:32:05,941 DEBUG [M:0;jenkins-hbase3:46449 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 145321bb47ef4b5483ee4a4b5a9e6535 2023-08-10T06:32:05,954 DEBUG [M:0;jenkins-hbase3:46449 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9f626d6bcc6840c5979a7b46f059bf6f is 1281, key is \x00\x00\x00\x00\x00\x00\x00\x19/proc:d/1691649045821/Put/seqid=0 2023-08-10T06:32:06,358 INFO [M:0;jenkins-hbase3:46449 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=325.49 KB at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9f626d6bcc6840c5979a7b46f059bf6f 2023-08-10T06:32:06,363 INFO [M:0;jenkins-hbase3:46449 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9f626d6bcc6840c5979a7b46f059bf6f 2023-08-10T06:32:06,363 DEBUG [M:0;jenkins-hbase3:46449 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9f626d6bcc6840c5979a7b46f059bf6f 2023-08-10T06:32:06,380 DEBUG [M:0;jenkins-hbase3:46449 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2da93606436e44f38ccad039be36f41a is 82, key is jenkins-hbase3.apache.org,40209,1691649026642/rs:state/1691649028539/Put/seqid=0 2023-08-10T06:32:06,786 INFO [M:0;jenkins-hbase3:46449 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2da93606436e44f38ccad039be36f41a 2023-08-10T06:32:06,791 DEBUG [M:0;jenkins-hbase3:46449 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2da93606436e44f38ccad039be36f41a 2023-08-10T06:32:06,793 DEBUG [M:0;jenkins-hbase3:46449 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/145321bb47ef4b5483ee4a4b5a9e6535 as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/145321bb47ef4b5483ee4a4b5a9e6535 2023-08-10T06:32:06,798 INFO [M:0;jenkins-hbase3:46449 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/145321bb47ef4b5483ee4a4b5a9e6535, entries=8, sequenceid=763, filesize=5.5 K 2023-08-10T06:32:06,799 DEBUG [M:0;jenkins-hbase3:46449 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9f626d6bcc6840c5979a7b46f059bf6f as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9f626d6bcc6840c5979a7b46f059bf6f 2023-08-10T06:32:06,804 INFO [M:0;jenkins-hbase3:46449 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9f626d6bcc6840c5979a7b46f059bf6f 2023-08-10T06:32:06,804 INFO [M:0;jenkins-hbase3:46449 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9f626d6bcc6840c5979a7b46f059bf6f, entries=83, sequenceid=763, filesize=26.0 K 2023-08-10T06:32:06,805 DEBUG [M:0;jenkins-hbase3:46449 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2da93606436e44f38ccad039be36f41a as hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2da93606436e44f38ccad039be36f41a 2023-08-10T06:32:06,809 INFO [M:0;jenkins-hbase3:46449 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:43579/user/jenkins/test-data/686e0b40-6d01-9bd3-857a-9ea043dced4a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2da93606436e44f38ccad039be36f41a, entries=1, sequenceid=763, filesize=5.1 K 2023-08-10T06:32:06,810 INFO [M:0;jenkins-hbase3:46449 {}] regionserver.HRegion(3022): Finished flush of dataSize ~326.08 KB/333909, heapSize ~389.77 KB/399120, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1292ms, sequenceid=763, compaction requested=false 2023-08-10T06:32:06,811 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-info-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 145321bb47ef4b5483ee4a4b5a9e6535 2023-08-10T06:32:06,815 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-proc-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 9f626d6bcc6840c5979a7b46f059bf6f 2023-08-10T06:32:06,817 DEBUG [StoreFileCloser-1595e783b53d99cd5eef43b6debb2682-rs-1 {}] hfile.PrefetchExecutor(137): Removing filename from the prefetched persistence list: 2da93606436e44f38ccad039be36f41a 2023-08-10T06:32:06,817 INFO [M:0;jenkins-hbase3:46449 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-08-10T06:32:06,817 DEBUG [M:0;jenkins-hbase3:46449 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-08-10T06:32:06,821 INFO [M:0;jenkins-hbase3:46449 {}] flush.MasterFlushTableProcedureManager(81): stop: server shutting down. 2023-08-10T06:32:06,821 INFO [M:0;jenkins-hbase3:46449 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-08-10T06:32:06,821 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-08-10T06:32:06,822 INFO [M:0;jenkins-hbase3:46449 {}] ipc.NettyRpcServer(213): Stopping server on /172.31.12.81:46449 2023-08-10T06:32:06,927 INFO [M:0;jenkins-hbase3:46449 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,46449,1691649023646; zookeeper connection closed. 2023-08-10T06:32:06,957 WARN [Listener at jenkins-hbase3.apache.org/40387 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-08-10T06:32:06,961 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-08-10T06:32:07,068 WARN [BP-1419340424-172.31.12.81-1691649020475 heartbeating to localhost/127.0.0.1:43579 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-08-10T06:32:07,068 WARN [BP-1419340424-172.31.12.81-1691649020475 heartbeating to localhost/127.0.0.1:43579 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1419340424-172.31.12.81-1691649020475 (Datanode Uuid 08fffa40-adf7-4a28-ba0e-d7b0211e926d) service to localhost/127.0.0.1:43579 2023-08-10T06:32:07,071 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/cluster_336bfe0c-247d-9555-0740-69ce104bc2cc/dfs/data/data1/current/BP-1419340424-172.31.12.81-1691649020475 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-08-10T06:32:07,072 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/17ec0433-0bea-25dd-6873-3251705bffe2/cluster_336bfe0c-247d-9555-0740-69ce104bc2cc/dfs/data/data2/current/BP-1419340424-172.31.12.81-1691649020475 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-08-10T06:32:07,098 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-08-10T06:32:07,275 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-08-10T06:32:07,275 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-08-10T06:32:07,286 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-08-10T06:32:07,401 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-08-10T06:32:07,513 ERROR [Thread[Thread-277,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-08-10T06:32:07,513 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-08-10T06:32:07,522 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-08-10T06:32:07,529 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-08-10T06:32:07,530 ERROR [Thread[Thread-287,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-08-10T06:32:07,535 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-08-10T06:32:07,638 ERROR [Thread[Thread-252,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-08-10T06:32:07,638 INFO [Listener at jenkins-hbase3.apache.org/40387 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped