2023-10-02 12:36:20,775 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 2023-10-02 12:36:20,793 main DEBUG Took 0.015153 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-10-02 12:36:20,793 main DEBUG PluginManager 'Core' found 129 plugins 2023-10-02 12:36:20,794 main DEBUG PluginManager 'Level' found 0 plugins 2023-10-02 12:36:20,795 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-10-02 12:36:20,797 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:36:20,806 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-10-02 12:36:20,822 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:36:20,824 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:36:20,824 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:36:20,825 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:36:20,825 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:36:20,826 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:36:20,827 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:36:20,827 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:36:20,828 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:36:20,828 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:36:20,829 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:36:20,829 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:36:20,830 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:36:20,830 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:36:20,831 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:36:20,831 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:36:20,832 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:36:20,832 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:36:20,833 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:36:20,833 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:36:20,834 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:36:20,834 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:36:20,835 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:36:20,835 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:36:20,835 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:36:20,835 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-10-02 12:36:20,838 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:36:20,840 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-10-02 12:36:20,842 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-10-02 12:36:20,843 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-10-02 12:36:20,844 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-10-02 12:36:20,844 main DEBUG PluginManager 'Converter' found 47 plugins 2023-10-02 12:36:20,860 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-10-02 12:36:20,863 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-10-02 12:36:20,868 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-10-02 12:36:20,869 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-10-02 12:36:20,869 main DEBUG createAppenders(={Console}) 2023-10-02 12:36:20,870 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 initialized 2023-10-02 12:36:20,870 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 2023-10-02 12:36:20,871 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 OK. 2023-10-02 12:36:20,871 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-10-02 12:36:20,872 main DEBUG OutputStream closed 2023-10-02 12:36:20,872 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-10-02 12:36:20,872 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-10-02 12:36:20,873 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@2145b572 OK 2023-10-02 12:36:20,923 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-10-02 12:36:20,925 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-10-02 12:36:20,926 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-10-02 12:36:20,928 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-10-02 12:36:20,928 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-10-02 12:36:20,928 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-10-02 12:36:20,929 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-10-02 12:36:20,929 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-10-02 12:36:20,929 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-10-02 12:36:20,930 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-10-02 12:36:20,930 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-10-02 12:36:20,930 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-10-02 12:36:20,930 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-10-02 12:36:20,931 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-10-02 12:36:20,931 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-10-02 12:36:20,931 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-10-02 12:36:20,932 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-10-02 12:36:20,933 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-10-02 12:36:20,937 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-10-02 12:36:20,937 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@49912c99) with optional ClassLoader: null 2023-10-02 12:36:20,937 main DEBUG Shutdown hook enabled. Registering a new one. 2023-10-02 12:36:20,938 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@49912c99] started OK. 2023-10-02T12:36:20,969 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestFullBackupWithFailures timeout: 13 mins 2023-10-02 12:36:20,974 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-10-02 12:36:20,974 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-10-02T12:36:21,362 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737 2023-10-02T12:36:21,368 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-10-02T12:36:21,370 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:36:21,371 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-10-02T12:36:21,375 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/cluster_523362f7-c993-0d07-9ba7-304dd19a6089, deleteOnExit=true 2023-10-02T12:36:21,375 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-10-02T12:36:21,375 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/test.cache.data in system properties and HBase conf 2023-10-02T12:36:21,376 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/hadoop.tmp.dir in system properties and HBase conf 2023-10-02T12:36:21,376 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/hadoop.log.dir in system properties and HBase conf 2023-10-02T12:36:21,377 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/mapreduce.cluster.local.dir in system properties and HBase conf 2023-10-02T12:36:21,377 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-10-02T12:36:21,377 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-10-02T12:36:21,481 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-10-02T12:36:22,093 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-10-02T12:36:22,101 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:36:22,101 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:36:22,103 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-10-02T12:36:22,104 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:36:22,104 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-10-02T12:36:22,105 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-10-02T12:36:22,105 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:36:22,106 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:36:22,106 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-10-02T12:36:22,107 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/nfs.dump.dir in system properties and HBase conf 2023-10-02T12:36:22,107 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/java.io.tmpdir in system properties and HBase conf 2023-10-02T12:36:22,107 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:36:22,108 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-10-02T12:36:22,108 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-10-02T12:36:22,753 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-10-02T12:36:22,759 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-10-02T12:36:23,142 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-10-02T12:36:23,340 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@6b8729fb via org.mortbay.log.Slf4jLog 2023-10-02T12:36:23,369 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:36:23,434 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:36:23,507 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/java.io.tmpdir/Jetty_localhost_45007_hdfs____.byc5ds/webapp 2023-10-02T12:36:23,822 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:45007 2023-10-02T12:36:23,851 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-10-02T12:36:23,851 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-10-02T12:36:24,666 WARN [Listener at localhost/40817 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:36:24,836 WARN [Listener at localhost/40817 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-10-02T12:36:24,870 WARN [Listener at localhost/40817 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:36:24,878 INFO [Listener at localhost/40817 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:36:24,894 INFO [Listener at localhost/40817 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/java.io.tmpdir/Jetty_localhost_33265_datanode____bw8zg8/webapp 2023-10-02T12:36:25,011 INFO [Listener at localhost/40817 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33265 2023-10-02T12:36:25,418 WARN [Listener at localhost/40383 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:36:25,873 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-10-02T12:36:26,204 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x81cc7f350f3f1225: Processing first storage report for DS-0c137cf7-1875-4349-b1e4-5237971c808a from datanode 0957471b-e404-4b45-bd6c-b720920e99f1 2023-10-02T12:36:26,207 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x81cc7f350f3f1225: from storage DS-0c137cf7-1875-4349-b1e4-5237971c808a node DatanodeRegistration(127.0.0.1:33097, datanodeUuid=0957471b-e404-4b45-bd6c-b720920e99f1, infoPort=37503, infoSecurePort=0, ipcPort=40383, storageInfo=lv=-57;cid=testClusterID;nsid=1310977224;c=1696250182869), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2023-10-02T12:36:26,207 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x81cc7f350f3f1225: Processing first storage report for DS-dddab04a-6b20-4531-8240-e339a726a762 from datanode 0957471b-e404-4b45-bd6c-b720920e99f1 2023-10-02T12:36:26,207 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x81cc7f350f3f1225: from storage DS-dddab04a-6b20-4531-8240-e339a726a762 node DatanodeRegistration(127.0.0.1:33097, datanodeUuid=0957471b-e404-4b45-bd6c-b720920e99f1, infoPort=37503, infoSecurePort=0, ipcPort=40383, storageInfo=lv=-57;cid=testClusterID;nsid=1310977224;c=1696250182869), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-10-02T12:36:26,255 DEBUG [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737 2023-10-02T12:36:26,389 INFO [Listener at localhost/40383 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/cluster_523362f7-c993-0d07-9ba7-304dd19a6089/zookeeper_0, clientPort=57296, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/cluster_523362f7-c993-0d07-9ba7-304dd19a6089/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/cluster_523362f7-c993-0d07-9ba7-304dd19a6089/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-10-02T12:36:26,409 INFO [Listener at localhost/40383 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=57296 2023-10-02T12:36:26,419 INFO [Listener at localhost/40383 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:36:26,421 INFO [Listener at localhost/40383 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:36:27,150 INFO [Listener at localhost/40383 {}] util.FSUtils(462): Created version file at hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09 with version=8 2023-10-02T12:36:27,151 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/hbase-staging 2023-10-02T12:36:27,175 DEBUG [Listener at localhost/40383 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-10-02T12:36:27,176 DEBUG [Listener at localhost/40383 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-10-02T12:36:27,176 DEBUG [Listener at localhost/40383 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-10-02T12:36:27,176 DEBUG [Listener at localhost/40383 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-10-02T12:36:27,465 DEBUG [Listener at localhost/40383 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-10-02T12:36:28,642 INFO [Listener at localhost/40383 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-10-02T12:36:28,728 INFO [Listener at localhost/40383 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:36:28,729 INFO [Listener at localhost/40383 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-10-02T12:36:28,729 INFO [Listener at localhost/40383 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-10-02T12:36:28,729 INFO [Listener at localhost/40383 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:36:28,729 INFO [Listener at localhost/40383 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-10-02T12:36:28,986 INFO [Listener at localhost/40383 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-10-02T12:36:29,209 INFO [Listener at localhost/40383 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-10-02T12:36:29,238 DEBUG [Listener at localhost/40383 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-10-02T12:36:29,246 INFO [Listener at localhost/40383 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-10-02T12:36:29,329 DEBUG [Listener at localhost/40383 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 14227 (auto-detected) 2023-10-02T12:36:29,331 DEBUG [Listener at localhost/40383 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-10-02T12:36:29,432 INFO [Listener at localhost/40383 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:36327 2023-10-02T12:36:29,455 INFO [Listener at localhost/40383 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:36:29,458 INFO [Listener at localhost/40383 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:36:29,495 INFO [Listener at localhost/40383 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:36327 connecting to ZooKeeper ensemble=127.0.0.1:57296 2023-10-02T12:36:29,583 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:363270x0, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-10-02T12:36:29,623 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:36327-0x101732d2adc0000 connected 2023-10-02T12:36:29,686 DEBUG [Listener at localhost/40383 {}] zookeeper.ZKUtil(113): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-10-02T12:36:29,690 DEBUG [Listener at localhost/40383 {}] zookeeper.ZKUtil(113): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-10-02T12:36:29,695 DEBUG [Listener at localhost/40383 {}] zookeeper.ZKUtil(113): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-10-02T12:36:29,712 DEBUG [Listener at localhost/40383 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36327 2023-10-02T12:36:29,714 DEBUG [Listener at localhost/40383 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36327 2023-10-02T12:36:29,718 DEBUG [Listener at localhost/40383 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36327 2023-10-02T12:36:29,726 DEBUG [Listener at localhost/40383 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36327 2023-10-02T12:36:29,726 DEBUG [Listener at localhost/40383 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36327 2023-10-02T12:36:29,810 INFO [Listener at localhost/40383 {}] log.Log(170): Logging initialized @9772ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-10-02T12:36:29,998 INFO [Listener at localhost/40383 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-10-02T12:36:29,999 INFO [Listener at localhost/40383 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-10-02T12:36:30,000 INFO [Listener at localhost/40383 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-10-02T12:36:30,002 INFO [Listener at localhost/40383 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-10-02T12:36:30,002 INFO [Listener at localhost/40383 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-10-02T12:36:30,002 INFO [Listener at localhost/40383 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-10-02T12:36:30,005 INFO [Listener at localhost/40383 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-10-02T12:36:30,166 INFO [Listener at localhost/40383 {}] http.HttpServer(1219): Jetty bound to port 38591 2023-10-02T12:36:30,169 INFO [Listener at localhost/40383 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:36:30,208 INFO [Listener at localhost/40383 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:36:30,212 INFO [Listener at localhost/40383 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@25efe20a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/hadoop.log.dir/,AVAILABLE} 2023-10-02T12:36:30,213 INFO [Listener at localhost/40383 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:36:30,214 INFO [Listener at localhost/40383 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@597b40c3{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-10-02T12:36:30,278 INFO [Listener at localhost/40383 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-10-02T12:36:30,295 INFO [Listener at localhost/40383 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-10-02T12:36:30,296 INFO [Listener at localhost/40383 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-10-02T12:36:30,298 INFO [Listener at localhost/40383 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-10-02T12:36:30,305 INFO [Listener at localhost/40383 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:36:31,489 INFO [Listener at localhost/40383 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@2afa8b4f{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-10-02T12:36:31,501 INFO [Listener at localhost/40383 {}] server.AbstractConnector(333): Started ServerConnector@5869c901{HTTP/1.1, (http/1.1)}{0.0.0.0:38591} 2023-10-02T12:36:31,501 INFO [Listener at localhost/40383 {}] server.Server(415): Started @11464ms 2023-10-02T12:36:31,505 INFO [Listener at localhost/40383 {}] master.HMaster(485): hbase.rootdir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09, hbase.cluster.distributed=false 2023-10-02T12:36:31,567 INFO [Listener at localhost/40383 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-10-02T12:36:31,567 INFO [Listener at localhost/40383 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:36:31,567 INFO [Listener at localhost/40383 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-10-02T12:36:31,567 INFO [Listener at localhost/40383 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-10-02T12:36:31,568 INFO [Listener at localhost/40383 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:36:31,568 INFO [Listener at localhost/40383 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-10-02T12:36:31,576 INFO [Listener at localhost/40383 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-10-02T12:36:31,578 INFO [Listener at localhost/40383 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-10-02T12:36:31,579 INFO [Listener at localhost/40383 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:37985 2023-10-02T12:36:31,582 INFO [Listener at localhost/40383 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-10-02T12:36:31,590 DEBUG [Listener at localhost/40383 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-10-02T12:36:31,591 INFO [Listener at localhost/40383 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:36:31,594 INFO [Listener at localhost/40383 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:36:31,596 INFO [Listener at localhost/40383 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:37985 connecting to ZooKeeper ensemble=127.0.0.1:57296 2023-10-02T12:36:31,625 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): regionserver:379850x0, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-10-02T12:36:31,629 DEBUG [Listener at localhost/40383 {}] zookeeper.ZKUtil(113): regionserver:379850x0, quorum=127.0.0.1:57296, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-10-02T12:36:31,629 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:37985-0x101732d2adc0001 connected 2023-10-02T12:36:31,631 DEBUG [Listener at localhost/40383 {}] zookeeper.ZKUtil(113): regionserver:37985-0x101732d2adc0001, quorum=127.0.0.1:57296, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-10-02T12:36:31,632 DEBUG [Listener at localhost/40383 {}] zookeeper.ZKUtil(113): regionserver:37985-0x101732d2adc0001, quorum=127.0.0.1:57296, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-10-02T12:36:31,632 DEBUG [Listener at localhost/40383 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37985 2023-10-02T12:36:31,633 DEBUG [Listener at localhost/40383 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37985 2023-10-02T12:36:31,639 DEBUG [Listener at localhost/40383 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37985 2023-10-02T12:36:31,640 DEBUG [Listener at localhost/40383 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37985 2023-10-02T12:36:31,640 DEBUG [Listener at localhost/40383 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37985 2023-10-02T12:36:31,643 INFO [Listener at localhost/40383 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-10-02T12:36:31,644 INFO [Listener at localhost/40383 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-10-02T12:36:31,644 INFO [Listener at localhost/40383 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-10-02T12:36:31,644 INFO [Listener at localhost/40383 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-10-02T12:36:31,645 INFO [Listener at localhost/40383 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-10-02T12:36:31,645 INFO [Listener at localhost/40383 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-10-02T12:36:31,645 INFO [Listener at localhost/40383 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-10-02T12:36:31,647 INFO [Listener at localhost/40383 {}] http.HttpServer(1219): Jetty bound to port 42915 2023-10-02T12:36:31,647 INFO [Listener at localhost/40383 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:36:31,650 INFO [Listener at localhost/40383 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:36:31,650 INFO [Listener at localhost/40383 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@7e8f64e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/hadoop.log.dir/,AVAILABLE} 2023-10-02T12:36:31,651 INFO [Listener at localhost/40383 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:36:31,651 INFO [Listener at localhost/40383 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@9fbde29{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-10-02T12:36:31,662 INFO [Listener at localhost/40383 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-10-02T12:36:31,663 INFO [Listener at localhost/40383 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-10-02T12:36:31,663 INFO [Listener at localhost/40383 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-10-02T12:36:31,663 INFO [Listener at localhost/40383 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-10-02T12:36:31,664 INFO [Listener at localhost/40383 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:36:31,669 INFO [Listener at localhost/40383 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@33da7094{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-10-02T12:36:31,670 INFO [Listener at localhost/40383 {}] server.AbstractConnector(333): Started ServerConnector@6e35d418{HTTP/1.1, (http/1.1)}{0.0.0.0:42915} 2023-10-02T12:36:31,670 INFO [Listener at localhost/40383 {}] server.Server(415): Started @11633ms 2023-10-02T12:36:31,675 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:36:31,677 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@26fbbebc{HTTP/1.1, (http/1.1)}{0.0.0.0:39119} 2023-10-02T12:36:31,678 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @11640ms 2023-10-02T12:36:31,678 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2388): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,36327,1696250187444 2023-10-02T12:36:31,690 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:36:31,690 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37985-0x101732d2adc0001, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:36:31,691 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,36327,1696250187444 2023-10-02T12:36:31,699 DEBUG [M:0;jenkins-hbase3:36327 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:36327 2023-10-02T12:36:31,709 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-10-02T12:36:31,710 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37985-0x101732d2adc0001, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-10-02T12:36:31,710 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:36:31,710 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37985-0x101732d2adc0001, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:36:31,711 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Set watcher on existing znode=/1/master 2023-10-02T12:36:31,712 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,36327,1696250187444 from backup master directory 2023-10-02T12:36:31,712 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Set watcher on existing znode=/1/master 2023-10-02T12:36:31,716 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,36327,1696250187444 2023-10-02T12:36:31,716 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37985-0x101732d2adc0001, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:36:31,716 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:36:31,717 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-10-02T12:36:31,717 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,36327,1696250187444 2023-10-02T12:36:31,722 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-10-02T12:36:31,724 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-10-02T12:36:31,816 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/hbase.id with ID: b646a0e8-bc77-4b87-8617-2d0b56c1d471 2023-10-02T12:36:31,860 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:36:31,876 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:36:31,876 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37985-0x101732d2adc0001, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:36:31,921 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:36:31,923 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-10-02T12:36:31,942 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-10-02T12:36:31,943 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-10-02T12:36:31,944 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:36:31,954 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:36:31,956 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:36:32,026 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/data/master/store 2023-10-02T12:36:32,453 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-10-02T12:36:32,454 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:36:32,454 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-10-02T12:36:32,454 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:36:32,455 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:36:32,455 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-10-02T12:36:32,455 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:36:32,455 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:36:32,455 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:36:32,457 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/data/master/store/.initializing 2023-10-02T12:36:32,458 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/WALs/jenkins-hbase3.apache.org,36327,1696250187444 2023-10-02T12:36:32,470 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-10-02T12:36:32,485 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C36327%2C1696250187444, suffix=, logDir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/WALs/jenkins-hbase3.apache.org,36327,1696250187444, archiveDir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/oldWALs, maxLogs=10 2023-10-02T12:36:32,513 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/WALs/jenkins-hbase3.apache.org,36327,1696250187444/jenkins-hbase3.apache.org%2C36327%2C1696250187444.1696250192492, exclude list is [], retry=0 2023-10-02T12:36:32,543 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33097,DS-0c137cf7-1875-4349-b1e4-5237971c808a,DISK] 2023-10-02T12:36:32,551 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:36:32,622 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/WALs/jenkins-hbase3.apache.org,36327,1696250187444/jenkins-hbase3.apache.org%2C36327%2C1696250187444.1696250192492 2023-10-02T12:36:32,622 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33097,DS-0c137cf7-1875-4349-b1e4-5237971c808a,DISK]] 2023-10-02T12:36:32,623 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:36:32,624 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:36:32,628 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:36:32,630 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:36:32,698 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:36:32,735 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-10-02T12:36:32,740 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:32,744 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:36:32,744 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:36:32,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-10-02T12:36:32,748 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:32,749 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:36:32,749 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:36:32,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-10-02T12:36:32,751 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:32,752 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:36:32,752 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:36:32,755 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-10-02T12:36:32,755 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:32,755 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:36:32,760 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:36:32,761 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:36:32,776 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-10-02T12:36:32,779 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:36:32,784 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:36:32,784 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11405791360, jitterRate=0.06224709749221802}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-10-02T12:36:32,791 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:36:32,792 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-10-02T12:36:32,828 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77673a2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:36:32,883 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(860): No meta location available on zookeeper, skip migrating... 2023-10-02T12:36:32,904 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-10-02T12:36:32,905 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-10-02T12:36:32,908 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-10-02T12:36:32,909 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-10-02T12:36:32,917 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 6 msec 2023-10-02T12:36:32,917 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-10-02T12:36:32,959 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-10-02T12:36:32,977 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-10-02T12:36:32,980 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-10-02T12:36:32,984 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-10-02T12:36:32,985 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-10-02T12:36:32,987 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-10-02T12:36:32,990 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-10-02T12:36:32,996 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-10-02T12:36:32,998 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-10-02T12:36:32,999 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-10-02T12:36:33,000 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-10-02T12:36:33,018 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-10-02T12:36:33,020 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-10-02T12:36:33,025 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-10-02T12:36:33,025 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37985-0x101732d2adc0001, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-10-02T12:36:33,026 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37985-0x101732d2adc0001, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:36:33,026 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:36:33,027 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(804): Active/primary master=jenkins-hbase3.apache.org,36327,1696250187444, sessionid=0x101732d2adc0000, setting cluster-up flag (Was=false) 2023-10-02T12:36:33,036 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-10-02T12:36:33,044 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:36:33,044 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37985-0x101732d2adc0001, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:36:33,050 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-10-02T12:36:33,051 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,36327,1696250187444 2023-10-02T12:36:33,056 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:36:33,056 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37985-0x101732d2adc0001, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:36:33,061 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-10-02T12:36:33,062 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,36327,1696250187444 2023-10-02T12:36:33,069 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:36:33,069 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37985-0x101732d2adc0001, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:36:33,075 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-10-02T12:36:33,076 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,36327,1696250187444 2023-10-02T12:36:33,187 DEBUG [RS:0;jenkins-hbase3:37985 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:37985 2023-10-02T12:36:33,189 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(997): ClusterId : b646a0e8-bc77-4b87-8617-2d0b56c1d471 2023-10-02T12:36:33,191 INFO [RS:0;jenkins-hbase3:37985 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-10-02T12:36:33,194 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-10-02T12:36:33,200 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-10-02T12:36:33,200 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-10-02T12:36:33,204 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-10-02T12:36:33,204 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-10-02T12:36:33,208 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-10-02T12:36:33,209 DEBUG [RS:0;jenkins-hbase3:37985 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@367d87fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:36:33,211 DEBUG [RS:0;jenkins-hbase3:37985 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bcdb6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-10-02T12:36:33,219 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-10-02T12:36:33,219 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-10-02T12:36:33,219 DEBUG [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-10-02T12:36:33,224 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,36327,1696250187444 with isa=jenkins-hbase3.apache.org/172.31.12.81:37985, startcode=1696250191566 2023-10-02T12:36:33,229 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-10-02T12:36:33,239 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-10-02T12:36:33,243 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-10-02T12:36:33,250 DEBUG [RS:0;jenkins-hbase3:37985 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-10-02T12:36:33,253 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,36327,1696250187444 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-10-02T12:36:33,258 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:36:33,258 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:36:33,258 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:36:33,258 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:36:33,259 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-10-02T12:36:33,259 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:36:33,259 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-10-02T12:36:33,259 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:36:33,266 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1696250223266 2023-10-02T12:36:33,271 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-10-02T12:36:33,272 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-10-02T12:36:33,273 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-10-02T12:36:33,275 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-10-02T12:36:33,278 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:33,279 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-10-02T12:36:33,289 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-10-02T12:36:33,290 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-10-02T12:36:33,291 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-10-02T12:36:33,292 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-10-02T12:36:33,292 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-10-02T12:36:33,295 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-10-02T12:36:33,299 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-10-02T12:36:33,301 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-10-02T12:36:33,302 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-10-02T12:36:33,315 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5ee8e008 to 127.0.0.1:57296 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:36:33,332 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61647d0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:36:33,332 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-10-02T12:36:33,337 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-10-02T12:36:33,337 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-10-02T12:36:33,340 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57609, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-10-02T12:36:33,342 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696250193339,5,FailOnTimeoutGroup] 2023-10-02T12:36:33,342 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696250193342,5,FailOnTimeoutGroup] 2023-10-02T12:36:33,342 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-10-02T12:36:33,342 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1635): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-10-02T12:36:33,345 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-10-02T12:36:33,345 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-10-02T12:36:33,356 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36327 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:33,389 DEBUG [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09 2023-10-02T12:36:33,389 DEBUG [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:40817 2023-10-02T12:36:33,389 DEBUG [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=38591 2023-10-02T12:36:33,398 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-10-02T12:36:33,398 DEBUG [RS:0;jenkins-hbase3:37985 {}] zookeeper.ZKUtil(111): regionserver:37985-0x101732d2adc0001, quorum=127.0.0.1:57296, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:33,398 WARN [RS:0;jenkins-hbase3:37985 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-10-02T12:36:33,399 INFO [RS:0;jenkins-hbase3:37985 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:36:33,399 DEBUG [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:33,404 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,37985,1696250191566] 2023-10-02T12:36:33,419 DEBUG [RS:0;jenkins-hbase3:37985 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-10-02T12:36:33,437 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-10-02T12:36:33,461 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-10-02T12:36:33,467 INFO [RS:0;jenkins-hbase3:37985 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-10-02T12:36:33,467 INFO [RS:0;jenkins-hbase3:37985 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:36:33,467 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-10-02T12:36:33,480 INFO [RS:0;jenkins-hbase3:37985 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-10-02T12:36:33,480 DEBUG [RS:0;jenkins-hbase3:37985 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:36:33,480 DEBUG [RS:0;jenkins-hbase3:37985 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:36:33,481 DEBUG [RS:0;jenkins-hbase3:37985 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:36:33,481 DEBUG [RS:0;jenkins-hbase3:37985 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:36:33,481 DEBUG [RS:0;jenkins-hbase3:37985 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:36:33,481 DEBUG [RS:0;jenkins-hbase3:37985 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-10-02T12:36:33,481 DEBUG [RS:0;jenkins-hbase3:37985 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:36:33,482 DEBUG [RS:0;jenkins-hbase3:37985 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:36:33,482 DEBUG [RS:0;jenkins-hbase3:37985 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:36:33,482 DEBUG [RS:0;jenkins-hbase3:37985 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:36:33,482 DEBUG [RS:0;jenkins-hbase3:37985 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:36:33,483 DEBUG [RS:0;jenkins-hbase3:37985 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-10-02T12:36:33,483 DEBUG [RS:0;jenkins-hbase3:37985 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-10-02T12:36:33,485 INFO [RS:0;jenkins-hbase3:37985 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-10-02T12:36:33,485 INFO [RS:0;jenkins-hbase3:37985 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-10-02T12:36:33,486 INFO [RS:0;jenkins-hbase3:37985 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-10-02T12:36:33,486 INFO [RS:0;jenkins-hbase3:37985 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-10-02T12:36:33,486 INFO [RS:0;jenkins-hbase3:37985 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37985,1696250191566-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-10-02T12:36:33,509 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-10-02T12:36:33,513 INFO [RS:0;jenkins-hbase3:37985 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,37985,1696250191566-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:36:33,536 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,37985,1696250191566 started 2023-10-02T12:36:33,536 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,37985,1696250191566, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:37985, sessionid=0x101732d2adc0001 2023-10-02T12:36:33,537 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-10-02T12:36:33,537 DEBUG [RS:0;jenkins-hbase3:37985 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:33,537 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,37985,1696250191566' 2023-10-02T12:36:33,537 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-10-02T12:36:33,538 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-10-02T12:36:33,539 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-10-02T12:36:33,539 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-10-02T12:36:33,539 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,37985,1696250191566' 2023-10-02T12:36:33,539 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:36:33,540 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:36:33,541 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-10-02T12:36:33,541 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-10-02T12:36:33,541 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-10-02T12:36:33,541 DEBUG [RS:0;jenkins-hbase3:37985 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:33,541 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,37985,1696250191566' 2023-10-02T12:36:33,541 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-10-02T12:36:33,542 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-10-02T12:36:33,543 DEBUG [RS:0;jenkins-hbase3:37985 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-10-02T12:36:33,543 INFO [RS:0;jenkins-hbase3:37985 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-10-02T12:36:33,543 INFO [RS:0;jenkins-hbase3:37985 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-10-02T12:36:33,656 INFO [RS:0;jenkins-hbase3:37985 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-10-02T12:36:33,660 INFO [RS:0;jenkins-hbase3:37985 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C37985%2C1696250191566, suffix=, logDir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566, archiveDir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/oldWALs, maxLogs=32 2023-10-02T12:36:33,678 DEBUG [RS:0;jenkins-hbase3:37985 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.1696250193662, exclude list is [], retry=0 2023-10-02T12:36:33,687 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33097,DS-0c137cf7-1875-4349-b1e4-5237971c808a,DISK] 2023-10-02T12:36:33,693 INFO [RS:0;jenkins-hbase3:37985 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.1696250193662 2023-10-02T12:36:33,694 DEBUG [RS:0;jenkins-hbase3:37985 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33097,DS-0c137cf7-1875-4349-b1e4-5237971c808a,DISK]] 2023-10-02T12:36:33,711 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-10-02T12:36:33,712 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09 2023-10-02T12:36:33,756 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:36:33,762 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-10-02T12:36:33,766 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-10-02T12:36:33,766 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:33,767 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:36:33,768 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-10-02T12:36:33,771 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-10-02T12:36:33,771 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:33,772 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:36:33,773 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-10-02T12:36:33,775 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-10-02T12:36:33,776 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:33,776 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:36:33,779 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/meta/1588230740 2023-10-02T12:36:33,779 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/meta/1588230740 2023-10-02T12:36:33,783 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-10-02T12:36:33,786 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-10-02T12:36:33,791 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:36:33,792 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9400027200, jitterRate=-0.12455424666404724}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-10-02T12:36:33,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-10-02T12:36:33,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-10-02T12:36:33,795 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-10-02T12:36:33,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-10-02T12:36:33,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-10-02T12:36:33,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-10-02T12:36:33,799 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-10-02T12:36:33,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-10-02T12:36:33,804 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-10-02T12:36:33,804 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-10-02T12:36:33,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-10-02T12:36:33,831 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-10-02T12:36:33,836 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-10-02T12:36:33,989 DEBUG [jenkins-hbase3:36327 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-10-02T12:36:33,998 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:34,006 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,37985,1696250191566, state=OPENING 2023-10-02T12:36:34,017 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-10-02T12:36:34,023 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37985-0x101732d2adc0001, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:36:34,023 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:36:34,024 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:36:34,024 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:36:34,027 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,37985,1696250191566}] 2023-10-02T12:36:34,223 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:34,226 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-10-02T12:36:34,230 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37146, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-10-02T12:36:34,245 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-10-02T12:36:34,247 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:36:34,247 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-10-02T12:36:34,251 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C37985%2C1696250191566.meta, suffix=.meta, logDir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566, archiveDir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/oldWALs, maxLogs=32 2023-10-02T12:36:34,267 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.meta.1696250194253.meta, exclude list is [], retry=0 2023-10-02T12:36:34,271 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33097,DS-0c137cf7-1875-4349-b1e4-5237971c808a,DISK] 2023-10-02T12:36:34,277 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.meta.1696250194253.meta 2023-10-02T12:36:34,277 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33097,DS-0c137cf7-1875-4349-b1e4-5237971c808a,DISK]] 2023-10-02T12:36:34,278 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:36:34,284 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:36:34,285 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-10-02T12:36:34,304 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-10-02T12:36:34,304 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-10-02T12:36:34,309 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-10-02T12:36:34,310 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:36:34,310 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-10-02T12:36:34,310 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-10-02T12:36:34,314 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-10-02T12:36:34,316 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-10-02T12:36:34,316 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:34,317 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:36:34,317 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-10-02T12:36:34,318 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-10-02T12:36:34,319 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:34,319 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:36:34,320 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-10-02T12:36:34,321 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-10-02T12:36:34,321 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:34,321 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:36:34,323 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/meta/1588230740 2023-10-02T12:36:34,326 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/meta/1588230740 2023-10-02T12:36:34,329 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-10-02T12:36:34,332 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-10-02T12:36:34,333 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11721483200, jitterRate=0.09164819121360779}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-10-02T12:36:34,335 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-10-02T12:36:34,350 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1696250194212 2023-10-02T12:36:34,374 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:34,375 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-10-02T12:36:34,376 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-10-02T12:36:34,378 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,37985,1696250191566, state=OPEN 2023-10-02T12:36:34,381 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-10-02T12:36:34,381 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37985-0x101732d2adc0001, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-10-02T12:36:34,381 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:36:34,381 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:36:34,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-10-02T12:36:34,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,37985,1696250191566 in 355 msec 2023-10-02T12:36:34,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-10-02T12:36:34,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 574 msec 2023-10-02T12:36:34,536 INFO [AsyncFSWAL-0-hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData {}] wal.AbstractFSWAL(1174): Slow sync cost: 133 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33097,DS-0c137cf7-1875-4349-b1e4-5237971c808a,DISK]] 2023-10-02T12:36:34,539 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.2520 sec 2023-10-02T12:36:34,539 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1063): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1696250194539, completionTime=-1 2023-10-02T12:36:34,539 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-10-02T12:36:34,539 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-10-02T12:36:34,620 DEBUG [hconnection-0x1500a07e-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:36:34,626 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37162, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:36:34,650 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-10-02T12:36:34,650 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1696250254650 2023-10-02T12:36:34,650 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1696250314650 2023-10-02T12:36:34,650 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 110 msec 2023-10-02T12:36:34,695 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,36327,1696250187444-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:36:34,696 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,36327,1696250187444-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:36:34,696 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,36327,1696250187444-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:36:34,698 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:36327, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:36:34,699 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-10-02T12:36:34,706 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-10-02T12:36:34,721 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-10-02T12:36:34,723 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2368): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-10-02T12:36:34,733 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-10-02T12:36:34,738 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:36:34,740 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:34,742 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:36:35,164 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 89f0eb5627b232df3a9a7c8d2e1d15a1, NAME => 'hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09 2023-10-02T12:36:35,177 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:36:35,177 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 89f0eb5627b232df3a9a7c8d2e1d15a1, disabling compactions & flushes 2023-10-02T12:36:35,177 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1. 2023-10-02T12:36:35,177 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1. 2023-10-02T12:36:35,177 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1. after waiting 0 ms 2023-10-02T12:36:35,177 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1. 2023-10-02T12:36:35,177 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1. 2023-10-02T12:36:35,177 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 89f0eb5627b232df3a9a7c8d2e1d15a1: 2023-10-02T12:36:35,180 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:36:35,199 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1696250195182"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250195182"}]},"ts":"1696250195182"} 2023-10-02T12:36:35,240 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:36:35,242 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:36:35,248 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250195242"}]},"ts":"1696250195242"} 2023-10-02T12:36:35,252 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-10-02T12:36:35,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=89f0eb5627b232df3a9a7c8d2e1d15a1, ASSIGN}] 2023-10-02T12:36:35,261 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=89f0eb5627b232df3a9a7c8d2e1d15a1, ASSIGN 2023-10-02T12:36:35,263 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=89f0eb5627b232df3a9a7c8d2e1d15a1, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37985,1696250191566; forceNewPlan=false, retain=false 2023-10-02T12:36:35,414 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=89f0eb5627b232df3a9a7c8d2e1d15a1, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:35,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 89f0eb5627b232df3a9a7c8d2e1d15a1, server=jenkins-hbase3.apache.org,37985,1696250191566}] 2023-10-02T12:36:35,580 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1. 2023-10-02T12:36:35,581 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 89f0eb5627b232df3a9a7c8d2e1d15a1, NAME => 'hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:36:35,582 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:36:35,583 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 89f0eb5627b232df3a9a7c8d2e1d15a1 2023-10-02T12:36:35,583 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:36:35,583 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 89f0eb5627b232df3a9a7c8d2e1d15a1 2023-10-02T12:36:35,583 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 89f0eb5627b232df3a9a7c8d2e1d15a1 2023-10-02T12:36:35,586 INFO [StoreOpener-89f0eb5627b232df3a9a7c8d2e1d15a1-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 89f0eb5627b232df3a9a7c8d2e1d15a1 2023-10-02T12:36:35,588 INFO [StoreOpener-89f0eb5627b232df3a9a7c8d2e1d15a1-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89f0eb5627b232df3a9a7c8d2e1d15a1 columnFamilyName info 2023-10-02T12:36:35,588 DEBUG [StoreOpener-89f0eb5627b232df3a9a7c8d2e1d15a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:35,589 INFO [StoreOpener-89f0eb5627b232df3a9a7c8d2e1d15a1-1 {}] regionserver.HStore(324): Store=89f0eb5627b232df3a9a7c8d2e1d15a1/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:36:35,591 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/namespace/89f0eb5627b232df3a9a7c8d2e1d15a1 2023-10-02T12:36:35,591 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/namespace/89f0eb5627b232df3a9a7c8d2e1d15a1 2023-10-02T12:36:35,595 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 89f0eb5627b232df3a9a7c8d2e1d15a1 2023-10-02T12:36:35,599 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/namespace/89f0eb5627b232df3a9a7c8d2e1d15a1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:36:35,600 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 89f0eb5627b232df3a9a7c8d2e1d15a1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12024432640, jitterRate=0.11986255645751953}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:36:35,601 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 89f0eb5627b232df3a9a7c8d2e1d15a1: 2023-10-02T12:36:35,603 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1., pid=6, masterSystemTime=1696250195573 2023-10-02T12:36:35,608 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1. 2023-10-02T12:36:35,608 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1. 2023-10-02T12:36:35,609 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=89f0eb5627b232df3a9a7c8d2e1d15a1, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:35,621 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-10-02T12:36:35,621 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 89f0eb5627b232df3a9a7c8d2e1d15a1, server=jenkins-hbase3.apache.org,37985,1696250191566 in 198 msec 2023-10-02T12:36:35,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-10-02T12:36:35,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=89f0eb5627b232df3a9a7c8d2e1d15a1, ASSIGN in 363 msec 2023-10-02T12:36:35,626 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:36:35,626 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250195626"}]},"ts":"1696250195626"} 2023-10-02T12:36:35,630 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-10-02T12:36:35,633 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:36:35,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 909 msec 2023-10-02T12:36:35,638 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-10-02T12:36:35,641 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-10-02T12:36:35,641 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): regionserver:37985-0x101732d2adc0001, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:36:35,641 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:36:35,695 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-10-02T12:36:35,715 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:36:35,720 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 35 msec 2023-10-02T12:36:35,730 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-10-02T12:36:35,743 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:36:35,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 18 msec 2023-10-02T12:36:35,758 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-10-02T12:36:35,762 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-10-02T12:36:35,762 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1193): Master has completed initialization 4.044sec 2023-10-02T12:36:35,764 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-10-02T12:36:35,766 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-10-02T12:36:35,767 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-10-02T12:36:35,768 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-10-02T12:36:35,768 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-10-02T12:36:35,770 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,36327,1696250187444-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-10-02T12:36:35,770 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,36327,1696250187444-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-10-02T12:36:35,785 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1296): Balancer post startup initialization complete, took 0 seconds 2023-10-02T12:36:35,787 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-10-02T12:36:35,843 DEBUG [Listener at localhost/40383 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x03667ac6 to 127.0.0.1:57296 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:36:35,848 DEBUG [Listener at localhost/40383 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ca2a7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:36:35,854 DEBUG [Listener at localhost/40383 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-10-02T12:36:35,854 DEBUG [Listener at localhost/40383 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-10-02T12:36:35,873 DEBUG [hconnection-0x70daa3a4-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:36:35,885 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37168, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:36:35,896 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,36327,1696250187444 2023-10-02T12:36:35,896 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-10-02T12:36:35,896 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/test.cache.data in system properties and HBase conf 2023-10-02T12:36:35,896 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/hadoop.tmp.dir in system properties and HBase conf 2023-10-02T12:36:35,896 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/hadoop.log.dir in system properties and HBase conf 2023-10-02T12:36:35,897 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/mapreduce.cluster.local.dir in system properties and HBase conf 2023-10-02T12:36:35,897 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-10-02T12:36:35,897 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-10-02T12:36:35,897 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:36:35,897 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:36:35,897 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-10-02T12:36:35,897 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:36:35,897 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-10-02T12:36:35,897 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-10-02T12:36:35,897 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:36:35,897 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:36:35,897 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-10-02T12:36:35,898 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/nfs.dump.dir in system properties and HBase conf 2023-10-02T12:36:35,898 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/java.io.tmpdir in system properties and HBase conf 2023-10-02T12:36:35,898 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:36:35,898 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-10-02T12:36:35,898 INFO [Listener at localhost/40383 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-10-02T12:36:37,702 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:36:38,152 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:36:38,158 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_37485_jobhistory____.a8c17t/webapp 2023-10-02T12:36:38,403 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-10-02T12:36:38,404 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-10-02T12:36:38,404 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:36:38,409 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:36:38,532 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:36:38,667 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:36:39,074 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-10-02T12:36:39,137 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:37485 2023-10-02T12:36:39,199 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-10-02T12:36:39,200 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-10-02T12:36:39,231 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-10-02T12:36:39,231 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-10-02T12:36:39,234 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:36:39,234 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-10-02T12:36:39,234 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-02T12:36:39,235 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-10-02T12:36:39,239 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-10-02T12:36:39,239 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-10-02T12:36:39,913 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:36:40,024 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-10-02T12:36:40,030 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-10-02T12:36:40,642 WARN [Listener at jenkins-hbase3.apache.org/34563 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:36:40,791 WARN [Listener at jenkins-hbase3.apache.org/34563 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:36:40,792 INFO [Listener at jenkins-hbase3.apache.org/34563 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:36:40,802 INFO [Listener at jenkins-hbase3.apache.org/34563 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_43811_cluster____vloutm/webapp 2023-10-02T12:36:40,926 INFO [Listener at jenkins-hbase3.apache.org/34563 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:36:40,926 INFO [Listener at jenkins-hbase3.apache.org/34563 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-10-02T12:36:40,926 INFO [Listener at jenkins-hbase3.apache.org/34563 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:36:40,927 INFO [Listener at jenkins-hbase3.apache.org/34563 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:36:40,943 INFO [Listener at jenkins-hbase3.apache.org/34563 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:36:41,326 INFO [Listener at jenkins-hbase3.apache.org/34563 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:36:41,536 INFO [Listener at jenkins-hbase3.apache.org/34563 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:36:41,558 INFO [Listener at jenkins-hbase3.apache.org/34563 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:43811 2023-10-02T12:36:42,406 WARN [Listener at jenkins-hbase3.apache.org/39651 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:36:42,452 WARN [Listener at jenkins-hbase3.apache.org/39651 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:36:42,452 INFO [Listener at jenkins-hbase3.apache.org/39651 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:36:42,462 INFO [Listener at jenkins-hbase3.apache.org/39651 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_34241_node____sfdxzh/webapp 2023-10-02T12:36:42,541 INFO [Listener at jenkins-hbase3.apache.org/39651 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-10-02T12:36:42,541 INFO [Listener at jenkins-hbase3.apache.org/39651 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:36:42,541 INFO [Listener at jenkins-hbase3.apache.org/39651 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:36:42,541 INFO [Listener at jenkins-hbase3.apache.org/39651 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:36:42,553 INFO [Listener at jenkins-hbase3.apache.org/39651 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:36:42,590 INFO [Listener at jenkins-hbase3.apache.org/39651 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:36:42,774 INFO [Listener at jenkins-hbase3.apache.org/39651 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:36:42,803 INFO [Listener at jenkins-hbase3.apache.org/39651 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:34241 2023-10-02T12:36:43,224 WARN [Listener at jenkins-hbase3.apache.org/34907 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:36:43,241 WARN [Listener at jenkins-hbase3.apache.org/34907 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:36:43,242 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:36:43,258 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_34637_node____hkqdrs/webapp 2023-10-02T12:36:43,338 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-10-02T12:36:43,338 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:36:43,338 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:36:43,339 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:36:43,350 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:36:43,374 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:36:43,551 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:36:43,609 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:34637 2023-10-02T12:36:43,710 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-10-02T12:36:43,710 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:40817/backupUT 2023-10-02T12:36:43,730 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:36:43,737 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53588, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:36:43,752 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-10-02T12:36:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-10-02T12:36:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-10-02T12:36:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-10-02T12:36:43,968 INFO [AsyncFSWAL-0-hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09 {}] wal.AbstractFSWAL(1174): Slow sync cost: 103 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:33097,DS-0c137cf7-1875-4349-b1e4-5237971c808a,DISK]] 2023-10-02T12:36:44,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-10-02T12:36:44,135 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:36:44,326 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 569 msec 2023-10-02T12:36:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-10-02T12:36:44,433 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-10-02T12:36:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-10-02T12:36:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-10-02T12:36:44,497 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:36:44,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 85 msec 2023-10-02T12:36:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-10-02T12:36:44,547 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-10-02T12:36:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-10-02T12:36:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-10-02T12:36:44,576 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:36:44,590 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 38 msec 2023-10-02T12:36:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-10-02T12:36:44,661 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-10-02T12:36:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-10-02T12:36:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-10-02T12:36:44,681 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:36:44,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 25 msec 2023-10-02T12:36:44,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-10-02T12:36:44,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'test-1696250203711', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:36:44,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1696250203711 2023-10-02T12:36:44,801 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1696250203711 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:36:44,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1696250203711" procId is: 13 2023-10-02T12:36:44,802 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:44,804 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1696250203711 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:36:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:36:44,889 INFO [RegionOpenAndInit-test-1696250203711-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 0d8a4463d318b78a2ee236eba02d8a78, NAME => 'test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1696250203711', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09 2023-10-02T12:36:44,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:36:44,934 DEBUG [RegionOpenAndInit-test-1696250203711-pool-0 {}] regionserver.HRegion(888): Instantiated test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:36:44,934 DEBUG [RegionOpenAndInit-test-1696250203711-pool-0 {}] regionserver.HRegion(1668): Closing 0d8a4463d318b78a2ee236eba02d8a78, disabling compactions & flushes 2023-10-02T12:36:44,935 INFO [RegionOpenAndInit-test-1696250203711-pool-0 {}] regionserver.HRegion(1690): Closing region test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. 2023-10-02T12:36:44,935 DEBUG [RegionOpenAndInit-test-1696250203711-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. 2023-10-02T12:36:44,935 DEBUG [RegionOpenAndInit-test-1696250203711-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. after waiting 0 ms 2023-10-02T12:36:44,935 DEBUG [RegionOpenAndInit-test-1696250203711-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. 2023-10-02T12:36:44,935 INFO [RegionOpenAndInit-test-1696250203711-pool-0 {}] regionserver.HRegion(1909): Closed test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. 2023-10-02T12:36:44,935 DEBUG [RegionOpenAndInit-test-1696250203711-pool-0 {}] regionserver.HRegion(1622): Region close journal for 0d8a4463d318b78a2ee236eba02d8a78: 2023-10-02T12:36:44,937 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1696250203711 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:36:44,938 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1696250204937"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250204937"}]},"ts":"1696250204937"} 2023-10-02T12:36:44,941 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:36:44,942 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1696250203711 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:36:44,943 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696250203711","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250204942"}]},"ts":"1696250204942"} 2023-10-02T12:36:44,945 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696250203711, state=ENABLING in hbase:meta 2023-10-02T12:36:44,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1696250203711, region=0d8a4463d318b78a2ee236eba02d8a78, ASSIGN}] 2023-10-02T12:36:44,963 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1696250203711, region=0d8a4463d318b78a2ee236eba02d8a78, ASSIGN 2023-10-02T12:36:44,964 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1696250203711, region=0d8a4463d318b78a2ee236eba02d8a78, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37985,1696250191566; forceNewPlan=false, retain=false 2023-10-02T12:36:45,116 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=0d8a4463d318b78a2ee236eba02d8a78, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:45,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:36:45,122 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 0d8a4463d318b78a2ee236eba02d8a78, server=jenkins-hbase3.apache.org,37985,1696250191566}] 2023-10-02T12:36:45,311 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. 2023-10-02T12:36:45,311 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 0d8a4463d318b78a2ee236eba02d8a78, NAME => 'test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:36:45,312 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:36:45,312 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1696250203711 0d8a4463d318b78a2ee236eba02d8a78 2023-10-02T12:36:45,312 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:36:45,312 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 0d8a4463d318b78a2ee236eba02d8a78 2023-10-02T12:36:45,312 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 0d8a4463d318b78a2ee236eba02d8a78 2023-10-02T12:36:45,323 INFO [StoreOpener-0d8a4463d318b78a2ee236eba02d8a78-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 0d8a4463d318b78a2ee236eba02d8a78 2023-10-02T12:36:45,326 INFO [StoreOpener-0d8a4463d318b78a2ee236eba02d8a78-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0d8a4463d318b78a2ee236eba02d8a78 columnFamilyName f 2023-10-02T12:36:45,326 DEBUG [StoreOpener-0d8a4463d318b78a2ee236eba02d8a78-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:45,327 INFO [StoreOpener-0d8a4463d318b78a2ee236eba02d8a78-1 {}] regionserver.HStore(324): Store=0d8a4463d318b78a2ee236eba02d8a78/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:36:45,329 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/default/test-1696250203711/0d8a4463d318b78a2ee236eba02d8a78 2023-10-02T12:36:45,329 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/default/test-1696250203711/0d8a4463d318b78a2ee236eba02d8a78 2023-10-02T12:36:45,336 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 0d8a4463d318b78a2ee236eba02d8a78 2023-10-02T12:36:45,344 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/default/test-1696250203711/0d8a4463d318b78a2ee236eba02d8a78/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:36:45,345 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 0d8a4463d318b78a2ee236eba02d8a78; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10046144480, jitterRate=-0.06437988579273224}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:36:45,346 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 0d8a4463d318b78a2ee236eba02d8a78: 2023-10-02T12:36:45,348 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78., pid=15, masterSystemTime=1696250205284 2023-10-02T12:36:45,352 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=0d8a4463d318b78a2ee236eba02d8a78, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:45,352 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. 2023-10-02T12:36:45,354 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. 2023-10-02T12:36:45,360 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-10-02T12:36:45,360 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 0d8a4463d318b78a2ee236eba02d8a78, server=jenkins-hbase3.apache.org,37985,1696250191566 in 234 msec 2023-10-02T12:36:45,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-10-02T12:36:45,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1696250203711, region=0d8a4463d318b78a2ee236eba02d8a78, ASSIGN in 402 msec 2023-10-02T12:36:45,365 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1696250203711 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:36:45,366 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696250203711","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250205365"}]},"ts":"1696250205365"} 2023-10-02T12:36:45,368 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696250203711, state=ENABLED in hbase:meta 2023-10-02T12:36:45,371 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1696250203711 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:36:45,375 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1696250203711 in 577 msec 2023-10-02T12:36:45,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:36:45,425 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:test-1696250203711, procId: 13 completed 2023-10-02T12:36:45,427 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4f398338 to 127.0.0.1:57296 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:36:45,441 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51bc0d59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:36:45,447 DEBUG [hconnection-0x108abdbc-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:36:45,451 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48218, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:36:45,461 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37985 {}] regionserver.HRegion(8210): writing data to region test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. with WAL disabled. Data may be lost in the event of a crash. 2023-10-02T12:36:45,589 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns2:test-16962502037111', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:36:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16962502037111 2023-10-02T12:36:45,594 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16962502037111 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:36:45,595 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:45,595 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16962502037111" procId is: 16 2023-10-02T12:36:45,597 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16962502037111 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:36:45,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:36:45,620 INFO [RegionOpenAndInit-ns2:test-16962502037111-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => b6da6887b5f5557b8eaf374a87ad833b, NAME => 'ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16962502037111', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09 2023-10-02T12:36:45,654 WARN [DataStreamer for file /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns2/test-16962502037111/b6da6887b5f5557b8eaf374a87ad833b/.regioninfo {}] hdfs.DataStreamer(982): Caught exception java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:1.8.0_362] at java.lang.Thread.join(Thread.java:1257) ~[?:1.8.0_362] at java.lang.Thread.join(Thread.java:1331) ~[?:1.8.0_362] at org.apache.hadoop.hdfs.DataStreamer.closeResponder(DataStreamer.java:980) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DataStreamer.endBlock(DataStreamer.java:630) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:807) ~[hadoop-hdfs-client-2.10.2.jar:?] 2023-10-02T12:36:45,655 DEBUG [RegionOpenAndInit-ns2:test-16962502037111-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:36:45,655 DEBUG [RegionOpenAndInit-ns2:test-16962502037111-pool-0 {}] regionserver.HRegion(1668): Closing b6da6887b5f5557b8eaf374a87ad833b, disabling compactions & flushes 2023-10-02T12:36:45,655 INFO [RegionOpenAndInit-ns2:test-16962502037111-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. 2023-10-02T12:36:45,655 DEBUG [RegionOpenAndInit-ns2:test-16962502037111-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. 2023-10-02T12:36:45,655 DEBUG [RegionOpenAndInit-ns2:test-16962502037111-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. after waiting 0 ms 2023-10-02T12:36:45,655 DEBUG [RegionOpenAndInit-ns2:test-16962502037111-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. 2023-10-02T12:36:45,655 INFO [RegionOpenAndInit-ns2:test-16962502037111-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. 2023-10-02T12:36:45,655 DEBUG [RegionOpenAndInit-ns2:test-16962502037111-pool-0 {}] regionserver.HRegion(1622): Region close journal for b6da6887b5f5557b8eaf374a87ad833b: 2023-10-02T12:36:45,659 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16962502037111 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:36:45,659 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696250205659"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250205659"}]},"ts":"1696250205659"} 2023-10-02T12:36:45,672 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:36:45,682 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16962502037111 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:36:45,683 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962502037111","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250205682"}]},"ts":"1696250205682"} 2023-10-02T12:36:45,691 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962502037111, state=ENABLING in hbase:meta 2023-10-02T12:36:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:36:45,719 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16962502037111, region=b6da6887b5f5557b8eaf374a87ad833b, ASSIGN}] 2023-10-02T12:36:45,729 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16962502037111, region=b6da6887b5f5557b8eaf374a87ad833b, ASSIGN 2023-10-02T12:36:45,735 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16962502037111, region=b6da6887b5f5557b8eaf374a87ad833b, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37985,1696250191566; forceNewPlan=false, retain=false 2023-10-02T12:36:45,887 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=b6da6887b5f5557b8eaf374a87ad833b, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:45,891 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure b6da6887b5f5557b8eaf374a87ad833b, server=jenkins-hbase3.apache.org,37985,1696250191566}] 2023-10-02T12:36:45,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:36:46,032 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1696250203711' 2023-10-02T12:36:46,052 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. 2023-10-02T12:36:46,052 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => b6da6887b5f5557b8eaf374a87ad833b, NAME => 'ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:36:46,052 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:36:46,053 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962502037111 b6da6887b5f5557b8eaf374a87ad833b 2023-10-02T12:36:46,053 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:36:46,053 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for b6da6887b5f5557b8eaf374a87ad833b 2023-10-02T12:36:46,053 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for b6da6887b5f5557b8eaf374a87ad833b 2023-10-02T12:36:46,056 INFO [StoreOpener-b6da6887b5f5557b8eaf374a87ad833b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region b6da6887b5f5557b8eaf374a87ad833b 2023-10-02T12:36:46,058 INFO [StoreOpener-b6da6887b5f5557b8eaf374a87ad833b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b6da6887b5f5557b8eaf374a87ad833b columnFamilyName f 2023-10-02T12:36:46,058 DEBUG [StoreOpener-b6da6887b5f5557b8eaf374a87ad833b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:46,059 INFO [StoreOpener-b6da6887b5f5557b8eaf374a87ad833b-1 {}] regionserver.HStore(324): Store=b6da6887b5f5557b8eaf374a87ad833b/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:36:46,061 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns2/test-16962502037111/b6da6887b5f5557b8eaf374a87ad833b 2023-10-02T12:36:46,062 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns2/test-16962502037111/b6da6887b5f5557b8eaf374a87ad833b 2023-10-02T12:36:46,071 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for b6da6887b5f5557b8eaf374a87ad833b 2023-10-02T12:36:46,085 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns2/test-16962502037111/b6da6887b5f5557b8eaf374a87ad833b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:36:46,087 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened b6da6887b5f5557b8eaf374a87ad833b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11023040800, jitterRate=0.0266006737947464}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:36:46,088 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for b6da6887b5f5557b8eaf374a87ad833b: 2023-10-02T12:36:46,092 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b., pid=18, masterSystemTime=1696250206045 2023-10-02T12:36:46,095 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. 2023-10-02T12:36:46,095 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. 2023-10-02T12:36:46,096 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=b6da6887b5f5557b8eaf374a87ad833b, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:46,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-10-02T12:36:46,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure b6da6887b5f5557b8eaf374a87ad833b, server=jenkins-hbase3.apache.org,37985,1696250191566 in 211 msec 2023-10-02T12:36:46,109 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-10-02T12:36:46,109 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16962502037111, region=b6da6887b5f5557b8eaf374a87ad833b, ASSIGN in 386 msec 2023-10-02T12:36:46,110 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16962502037111 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:36:46,110 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962502037111","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250206110"}]},"ts":"1696250206110"} 2023-10-02T12:36:46,115 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962502037111, state=ENABLED in hbase:meta 2023-10-02T12:36:46,119 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16962502037111 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:36:46,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16962502037111 in 530 msec 2023-10-02T12:36:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:36:46,214 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns2:test-16962502037111, procId: 16 completed 2023-10-02T12:36:46,221 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37985 {}] regionserver.HRegion(8210): writing data to region ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. with WAL disabled. Data may be lost in the event of a crash. 2023-10-02T12:36:46,390 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns3:test-16962502037112', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:36:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16962502037112 2023-10-02T12:36:46,396 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16962502037112 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:36:46,396 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:46,397 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16962502037112" procId is: 19 2023-10-02T12:36:46,398 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16962502037112 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:36:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:36:46,462 INFO [RegionOpenAndInit-ns3:test-16962502037112-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 1668fd8ee9c956d5c525bdda4f6f5b90, NAME => 'ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16962502037112', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09 2023-10-02T12:36:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:36:46,528 DEBUG [RegionOpenAndInit-ns3:test-16962502037112-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:36:46,528 DEBUG [RegionOpenAndInit-ns3:test-16962502037112-pool-0 {}] regionserver.HRegion(1668): Closing 1668fd8ee9c956d5c525bdda4f6f5b90, disabling compactions & flushes 2023-10-02T12:36:46,528 INFO [RegionOpenAndInit-ns3:test-16962502037112-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90. 2023-10-02T12:36:46,528 DEBUG [RegionOpenAndInit-ns3:test-16962502037112-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90. 2023-10-02T12:36:46,528 DEBUG [RegionOpenAndInit-ns3:test-16962502037112-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90. after waiting 0 ms 2023-10-02T12:36:46,528 DEBUG [RegionOpenAndInit-ns3:test-16962502037112-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90. 2023-10-02T12:36:46,528 INFO [RegionOpenAndInit-ns3:test-16962502037112-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90. 2023-10-02T12:36:46,528 DEBUG [RegionOpenAndInit-ns3:test-16962502037112-pool-0 {}] regionserver.HRegion(1622): Region close journal for 1668fd8ee9c956d5c525bdda4f6f5b90: 2023-10-02T12:36:46,531 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16962502037112 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:36:46,531 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696250206531"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250206531"}]},"ts":"1696250206531"} 2023-10-02T12:36:46,534 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:36:46,538 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16962502037112 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:36:46,538 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962502037112","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250206538"}]},"ts":"1696250206538"} 2023-10-02T12:36:46,541 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962502037112, state=ENABLING in hbase:meta 2023-10-02T12:36:46,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16962502037112, region=1668fd8ee9c956d5c525bdda4f6f5b90, ASSIGN}] 2023-10-02T12:36:46,557 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16962502037112, region=1668fd8ee9c956d5c525bdda4f6f5b90, ASSIGN 2023-10-02T12:36:46,559 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16962502037112, region=1668fd8ee9c956d5c525bdda4f6f5b90, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37985,1696250191566; forceNewPlan=false, retain=false 2023-10-02T12:36:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:36:46,711 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=1668fd8ee9c956d5c525bdda4f6f5b90, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:46,714 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 1668fd8ee9c956d5c525bdda4f6f5b90, server=jenkins-hbase3.apache.org,37985,1696250191566}] 2023-10-02T12:36:46,878 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90. 2023-10-02T12:36:46,878 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 1668fd8ee9c956d5c525bdda4f6f5b90, NAME => 'ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:36:46,879 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:36:46,879 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962502037112 1668fd8ee9c956d5c525bdda4f6f5b90 2023-10-02T12:36:46,879 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:36:46,879 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 1668fd8ee9c956d5c525bdda4f6f5b90 2023-10-02T12:36:46,879 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 1668fd8ee9c956d5c525bdda4f6f5b90 2023-10-02T12:36:46,884 INFO [StoreOpener-1668fd8ee9c956d5c525bdda4f6f5b90-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 1668fd8ee9c956d5c525bdda4f6f5b90 2023-10-02T12:36:46,887 INFO [StoreOpener-1668fd8ee9c956d5c525bdda4f6f5b90-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1668fd8ee9c956d5c525bdda4f6f5b90 columnFamilyName f 2023-10-02T12:36:46,887 DEBUG [StoreOpener-1668fd8ee9c956d5c525bdda4f6f5b90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:46,888 INFO [StoreOpener-1668fd8ee9c956d5c525bdda4f6f5b90-1 {}] regionserver.HStore(324): Store=1668fd8ee9c956d5c525bdda4f6f5b90/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:36:46,890 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns3/test-16962502037112/1668fd8ee9c956d5c525bdda4f6f5b90 2023-10-02T12:36:46,891 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns3/test-16962502037112/1668fd8ee9c956d5c525bdda4f6f5b90 2023-10-02T12:36:46,895 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 1668fd8ee9c956d5c525bdda4f6f5b90 2023-10-02T12:36:46,899 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns3/test-16962502037112/1668fd8ee9c956d5c525bdda4f6f5b90/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:36:46,900 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 1668fd8ee9c956d5c525bdda4f6f5b90; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11657958720, jitterRate=0.08573201298713684}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:36:46,901 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 1668fd8ee9c956d5c525bdda4f6f5b90: 2023-10-02T12:36:46,902 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90., pid=21, masterSystemTime=1696250206869 2023-10-02T12:36:46,907 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=1668fd8ee9c956d5c525bdda4f6f5b90, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:46,907 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90. 2023-10-02T12:36:46,907 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90. 2023-10-02T12:36:46,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-10-02T12:36:46,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 1668fd8ee9c956d5c525bdda4f6f5b90, server=jenkins-hbase3.apache.org,37985,1696250191566 in 195 msec 2023-10-02T12:36:46,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-10-02T12:36:46,916 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16962502037112, region=1668fd8ee9c956d5c525bdda4f6f5b90, ASSIGN in 365 msec 2023-10-02T12:36:46,917 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16962502037112 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:36:46,917 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962502037112","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250206917"}]},"ts":"1696250206917"} 2023-10-02T12:36:46,920 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962502037112, state=ENABLED in hbase:meta 2023-10-02T12:36:46,927 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16962502037112 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:36:46,936 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16962502037112 in 536 msec 2023-10-02T12:36:47,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:36:47,008 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns3:test-16962502037112, procId: 19 completed 2023-10-02T12:36:47,008 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16962502037112 get assigned. Timeout = 60000ms 2023-10-02T12:36:47,011 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:36:47,016 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16962502037112 assigned to meta. Checking AM states. 2023-10-02T12:36:47,017 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:36:47,017 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16962502037112 assigned. 2023-10-02T12:36:47,020 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns4:test-16962502037113', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:36:47,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16962502037113 2023-10-02T12:36:47,028 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16962502037113 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:36:47,028 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:47,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16962502037113" procId is: 22 2023-10-02T12:36:47,030 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16962502037113 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:36:47,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:36:47,055 INFO [RegionOpenAndInit-ns4:test-16962502037113-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 08d5f9946b11fd88451ddcf204fe133f, NAME => 'ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16962502037113', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09 2023-10-02T12:36:47,082 DEBUG [RegionOpenAndInit-ns4:test-16962502037113-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:36:47,082 DEBUG [RegionOpenAndInit-ns4:test-16962502037113-pool-0 {}] regionserver.HRegion(1668): Closing 08d5f9946b11fd88451ddcf204fe133f, disabling compactions & flushes 2023-10-02T12:36:47,082 INFO [RegionOpenAndInit-ns4:test-16962502037113-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f. 2023-10-02T12:36:47,082 DEBUG [RegionOpenAndInit-ns4:test-16962502037113-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f. 2023-10-02T12:36:47,082 DEBUG [RegionOpenAndInit-ns4:test-16962502037113-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f. after waiting 0 ms 2023-10-02T12:36:47,082 DEBUG [RegionOpenAndInit-ns4:test-16962502037113-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f. 2023-10-02T12:36:47,082 INFO [RegionOpenAndInit-ns4:test-16962502037113-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f. 2023-10-02T12:36:47,082 DEBUG [RegionOpenAndInit-ns4:test-16962502037113-pool-0 {}] regionserver.HRegion(1622): Region close journal for 08d5f9946b11fd88451ddcf204fe133f: 2023-10-02T12:36:47,084 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16962502037113 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:36:47,085 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696250207084"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250207084"}]},"ts":"1696250207084"} 2023-10-02T12:36:47,088 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:36:47,089 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16962502037113 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:36:47,089 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16962502037113","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250207089"}]},"ts":"1696250207089"} 2023-10-02T12:36:47,091 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16962502037113, state=ENABLING in hbase:meta 2023-10-02T12:36:47,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16962502037113, region=08d5f9946b11fd88451ddcf204fe133f, ASSIGN}] 2023-10-02T12:36:47,103 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16962502037113, region=08d5f9946b11fd88451ddcf204fe133f, ASSIGN 2023-10-02T12:36:47,105 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16962502037113, region=08d5f9946b11fd88451ddcf204fe133f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37985,1696250191566; forceNewPlan=false, retain=false 2023-10-02T12:36:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:36:47,257 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=08d5f9946b11fd88451ddcf204fe133f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:47,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 08d5f9946b11fd88451ddcf204fe133f, server=jenkins-hbase3.apache.org,37985,1696250191566}] 2023-10-02T12:36:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:36:47,422 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f. 2023-10-02T12:36:47,423 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 08d5f9946b11fd88451ddcf204fe133f, NAME => 'ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:36:47,423 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:36:47,423 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962502037113 08d5f9946b11fd88451ddcf204fe133f 2023-10-02T12:36:47,424 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:36:47,424 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 08d5f9946b11fd88451ddcf204fe133f 2023-10-02T12:36:47,424 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 08d5f9946b11fd88451ddcf204fe133f 2023-10-02T12:36:47,426 INFO [StoreOpener-08d5f9946b11fd88451ddcf204fe133f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 08d5f9946b11fd88451ddcf204fe133f 2023-10-02T12:36:47,428 INFO [StoreOpener-08d5f9946b11fd88451ddcf204fe133f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08d5f9946b11fd88451ddcf204fe133f columnFamilyName f 2023-10-02T12:36:47,428 DEBUG [StoreOpener-08d5f9946b11fd88451ddcf204fe133f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:47,429 INFO [StoreOpener-08d5f9946b11fd88451ddcf204fe133f-1 {}] regionserver.HStore(324): Store=08d5f9946b11fd88451ddcf204fe133f/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:36:47,430 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns4/test-16962502037113/08d5f9946b11fd88451ddcf204fe133f 2023-10-02T12:36:47,431 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns4/test-16962502037113/08d5f9946b11fd88451ddcf204fe133f 2023-10-02T12:36:47,435 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 08d5f9946b11fd88451ddcf204fe133f 2023-10-02T12:36:47,438 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns4/test-16962502037113/08d5f9946b11fd88451ddcf204fe133f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:36:47,439 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 08d5f9946b11fd88451ddcf204fe133f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12002624320, jitterRate=0.11783149838447571}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:36:47,440 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 08d5f9946b11fd88451ddcf204fe133f: 2023-10-02T12:36:47,441 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f., pid=24, masterSystemTime=1696250207416 2023-10-02T12:36:47,447 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f. 2023-10-02T12:36:47,447 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f. 2023-10-02T12:36:47,450 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=08d5f9946b11fd88451ddcf204fe133f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:47,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-10-02T12:36:47,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 08d5f9946b11fd88451ddcf204fe133f, server=jenkins-hbase3.apache.org,37985,1696250191566 in 192 msec 2023-10-02T12:36:47,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-10-02T12:36:47,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16962502037113, region=08d5f9946b11fd88451ddcf204fe133f, ASSIGN in 356 msec 2023-10-02T12:36:47,460 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16962502037113 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:36:47,461 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16962502037113","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250207460"}]},"ts":"1696250207460"} 2023-10-02T12:36:47,463 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16962502037113, state=ENABLED in hbase:meta 2023-10-02T12:36:47,466 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16962502037113 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:36:47,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16962502037113 in 446 msec 2023-10-02T12:36:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:36:47,642 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns4:test-16962502037113, procId: 22 completed 2023-10-02T12:36:47,642 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16962502037113 get assigned. Timeout = 60000ms 2023-10-02T12:36:47,642 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:36:47,648 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16962502037113 assigned to meta. Checking AM states. 2023-10-02T12:36:47,648 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:36:47,648 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16962502037113 assigned. 2023-10-02T12:36:47,649 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4f398338 to 127.0.0.1:57296 2023-10-02T12:36:47,649 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:36:47,709 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] hbase.ResourceChecker(147): before: backup.TestFullBackupWithFailures#testFullBackupWithFailures Thread=584, OpenFileDescriptor=660, MaxFileDescriptor=60000, SystemLoadAverage=513, ProcessCount=181, AvailableMemoryMB=5038 2023-10-02T12:36:47,710 WARN [Listener at jenkins-hbase3.apache.org/34907 {}] hbase.ResourceChecker(130): Thread=584 is superior to 500 2023-10-02T12:36:47,715 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] backup.TestFullBackupWithFailures(54): Running stage 0 2023-10-02T12:36:47,747 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-10-02T12:36:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-10-02T12:36:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-10-02T12:36:47,777 DEBUG [Listener at localhost/40383-EventThread {}] zookeeper.ZKWatcher(604): master:36327-0x101732d2adc0000, quorum=127.0.0.1:57296, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:36:47,790 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 38 msec 2023-10-02T12:36:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-10-02T12:36:47,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:36:47,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-10-02T12:36:47,904 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:36:47,904 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:47,905 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-10-02T12:36:47,907 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:36:47,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:36:47,933 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09 2023-10-02T12:36:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:36:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:36:48,356 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:36:48,357 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 232c4115fe6f97bf9747948e2cf60e81, disabling compactions & flushes 2023-10-02T12:36:48,357 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:48,357 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:48,357 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. after waiting 0 ms 2023-10-02T12:36:48,357 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:48,357 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:48,357 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 232c4115fe6f97bf9747948e2cf60e81: 2023-10-02T12:36:48,360 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:36:48,360 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1696250208360"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250208360"}]},"ts":"1696250208360"} 2023-10-02T12:36:48,363 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:36:48,365 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:36:48,365 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250208365"}]},"ts":"1696250208365"} 2023-10-02T12:36:48,369 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-10-02T12:36:48,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN}] 2023-10-02T12:36:48,382 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN 2023-10-02T12:36:48,384 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37985,1696250191566; forceNewPlan=false, retain=false 2023-10-02T12:36:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:36:48,536 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:48,539 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566}] 2023-10-02T12:36:48,699 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:48,700 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:36:48,701 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:36:48,701 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:48,701 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:36:48,701 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:48,701 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:48,705 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:48,712 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 232c4115fe6f97bf9747948e2cf60e81 columnFamilyName meta 2023-10-02T12:36:48,713 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:48,714 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(324): Store=232c4115fe6f97bf9747948e2cf60e81/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:36:48,714 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:48,717 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 232c4115fe6f97bf9747948e2cf60e81 columnFamilyName session 2023-10-02T12:36:48,717 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:48,718 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(324): Store=232c4115fe6f97bf9747948e2cf60e81/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:36:48,719 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:48,720 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:48,722 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:36:48,724 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:48,732 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:36:48,734 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened 232c4115fe6f97bf9747948e2cf60e81; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9675266880, jitterRate=-0.0989205539226532}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:36:48,735 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for 232c4115fe6f97bf9747948e2cf60e81: 2023-10-02T12:36:48,737 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81., pid=28, masterSystemTime=1696250208693 2023-10-02T12:36:48,744 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:48,745 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:48,745 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:48,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-10-02T12:36:48,751 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566 in 209 msec 2023-10-02T12:36:48,756 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-10-02T12:36:48,756 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN in 374 msec 2023-10-02T12:36:48,758 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:36:48,758 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250208758"}]},"ts":"1696250208758"} 2023-10-02T12:36:48,760 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-10-02T12:36:48,780 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:36:48,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 886 msec 2023-10-02T12:36:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:36:49,027 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-10-02T12:36:49,051 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:36:49,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-10-02T12:36:49,058 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:36:49,058 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:49,058 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-10-02T12:36:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:36:49,060 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:36:49,084 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => d15e6e23ef0ca9fe47b3a30aa65c34b4, NAME => 'backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09 2023-10-02T12:36:49,118 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:36:49,119 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing d15e6e23ef0ca9fe47b3a30aa65c34b4, disabling compactions & flushes 2023-10-02T12:36:49,119 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4. 2023-10-02T12:36:49,119 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4. 2023-10-02T12:36:49,119 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4. after waiting 0 ms 2023-10-02T12:36:49,119 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4. 2023-10-02T12:36:49,119 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4. 2023-10-02T12:36:49,119 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for d15e6e23ef0ca9fe47b3a30aa65c34b4: 2023-10-02T12:36:49,121 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:36:49,122 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1696250209121"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250209121"}]},"ts":"1696250209121"} 2023-10-02T12:36:49,131 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:36:49,133 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:36:49,134 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250209133"}]},"ts":"1696250209133"} 2023-10-02T12:36:49,136 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-10-02T12:36:49,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=d15e6e23ef0ca9fe47b3a30aa65c34b4, ASSIGN}] 2023-10-02T12:36:49,145 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=d15e6e23ef0ca9fe47b3a30aa65c34b4, ASSIGN 2023-10-02T12:36:49,147 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=d15e6e23ef0ca9fe47b3a30aa65c34b4, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,37985,1696250191566; forceNewPlan=false, retain=false 2023-10-02T12:36:49,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:36:49,199 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-10-02T12:36:49,199 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-10-02T12:36:49,200 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16962502037112 2023-10-02T12:36:49,200 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16962502037112 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:36:49,202 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1696250203711 2023-10-02T12:36:49,202 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1696250203711 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:36:49,203 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16962502037113 2023-10-02T12:36:49,203 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16962502037113 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:36:49,210 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16962502037111 2023-10-02T12:36:49,210 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16962502037111 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:36:49,299 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=d15e6e23ef0ca9fe47b3a30aa65c34b4, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:49,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure d15e6e23ef0ca9fe47b3a30aa65c34b4, server=jenkins-hbase3.apache.org,37985,1696250191566}] 2023-10-02T12:36:49,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:36:49,463 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4. 2023-10-02T12:36:49,464 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => d15e6e23ef0ca9fe47b3a30aa65c34b4, NAME => 'backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:36:49,464 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:36:49,464 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk d15e6e23ef0ca9fe47b3a30aa65c34b4 2023-10-02T12:36:49,464 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:36:49,464 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for d15e6e23ef0ca9fe47b3a30aa65c34b4 2023-10-02T12:36:49,464 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for d15e6e23ef0ca9fe47b3a30aa65c34b4 2023-10-02T12:36:49,467 INFO [StoreOpener-d15e6e23ef0ca9fe47b3a30aa65c34b4-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region d15e6e23ef0ca9fe47b3a30aa65c34b4 2023-10-02T12:36:49,469 INFO [StoreOpener-d15e6e23ef0ca9fe47b3a30aa65c34b4-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d15e6e23ef0ca9fe47b3a30aa65c34b4 columnFamilyName meta 2023-10-02T12:36:49,469 DEBUG [StoreOpener-d15e6e23ef0ca9fe47b3a30aa65c34b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:49,470 INFO [StoreOpener-d15e6e23ef0ca9fe47b3a30aa65c34b4-1 {}] regionserver.HStore(324): Store=d15e6e23ef0ca9fe47b3a30aa65c34b4/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:36:49,470 INFO [StoreOpener-d15e6e23ef0ca9fe47b3a30aa65c34b4-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region d15e6e23ef0ca9fe47b3a30aa65c34b4 2023-10-02T12:36:49,472 INFO [StoreOpener-d15e6e23ef0ca9fe47b3a30aa65c34b4-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d15e6e23ef0ca9fe47b3a30aa65c34b4 columnFamilyName session 2023-10-02T12:36:49,472 DEBUG [StoreOpener-d15e6e23ef0ca9fe47b3a30aa65c34b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:49,472 INFO [StoreOpener-d15e6e23ef0ca9fe47b3a30aa65c34b4-1 {}] regionserver.HStore(324): Store=d15e6e23ef0ca9fe47b3a30aa65c34b4/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:36:49,474 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system_bulk/d15e6e23ef0ca9fe47b3a30aa65c34b4 2023-10-02T12:36:49,475 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system_bulk/d15e6e23ef0ca9fe47b3a30aa65c34b4 2023-10-02T12:36:49,477 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:36:49,487 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for d15e6e23ef0ca9fe47b3a30aa65c34b4 2023-10-02T12:36:49,490 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system_bulk/d15e6e23ef0ca9fe47b3a30aa65c34b4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:36:49,491 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened d15e6e23ef0ca9fe47b3a30aa65c34b4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9719659520, jitterRate=-0.09478616714477539}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:36:49,492 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for d15e6e23ef0ca9fe47b3a30aa65c34b4: 2023-10-02T12:36:49,493 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4., pid=31, masterSystemTime=1696250209458 2023-10-02T12:36:49,496 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4. 2023-10-02T12:36:49,496 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4. 2023-10-02T12:36:49,497 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=d15e6e23ef0ca9fe47b3a30aa65c34b4, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:49,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-10-02T12:36:49,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure d15e6e23ef0ca9fe47b3a30aa65c34b4, server=jenkins-hbase3.apache.org,37985,1696250191566 in 196 msec 2023-10-02T12:36:49,508 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=30, resume processing ppid=29 2023-10-02T12:36:49,508 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=d15e6e23ef0ca9fe47b3a30aa65c34b4, ASSIGN in 360 msec 2023-10-02T12:36:49,509 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:36:49,510 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250209509"}]},"ts":"1696250209509"} 2023-10-02T12:36:49,514 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-10-02T12:36:49,521 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:36:49,523 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 470 msec 2023-10-02T12:36:49,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:36:49,668 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-10-02T12:36:49,756 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e7dc0d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:36:49,758 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:36:49,760 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53592, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:36:49,767 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:36:49,770 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48228, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:36:49,894 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:36:49,953 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:36:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250209953 (current time:1696250209953). 2023-10-02T12:36:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:36:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:36:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@187097f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:36:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:36:49,980 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48232, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:36:49,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:36:49,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:36:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:36:50,022 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:36:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-10-02T12:36:50,032 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:36:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:36:50,059 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot, current working dir path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:36:50,069 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:36:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:36:50,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:36:50,537 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:36:50,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 232c4115fe6f97bf9747948e2cf60e81}] 2023-10-02T12:36:50,552 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:36:50,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37985 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-10-02T12:36:50,715 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:50,717 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing 232c4115fe6f97bf9747948e2cf60e81 2/2 column families, dataSize=45 B heapSize=632 B 2023-10-02T12:36:50,843 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/547c8c6158404697bd5ec227f6f3d646 is 49, key is activesession:/session:c/1696250209934/Put/seqid=0 2023-10-02T12:36:50,876 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/547c8c6158404697bd5ec227f6f3d646 2023-10-02T12:36:50,936 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:36:51,008 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/547c8c6158404697bd5ec227f6f3d646 as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646 2023-10-02T12:36:51,024 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646, entries=1, sequenceid=5, filesize=4.9 K 2023-10-02T12:36:51,031 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 232c4115fe6f97bf9747948e2cf60e81 in 314ms, sequenceid=5, compaction requested=false 2023-10-02T12:36:51,031 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-10-02T12:36:51,033 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for 232c4115fe6f97bf9747948e2cf60e81: 2023-10-02T12:36:51,034 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. for snapshot_backup_system completed. 2023-10-02T12:36:51,035 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:36:51,052 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:36:51,061 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:36:51,062 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646] hfiles 2023-10-02T12:36:51,062 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646 for snapshot=snapshot_backup_system 2023-10-02T12:36:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:36:51,530 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:51,535 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-10-02T12:36:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster(3995): Remote procedure done, pid=33 2023-10-02T12:36:51,539 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:51,541 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:51,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-10-02T12:36:51,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure 232c4115fe6f97bf9747948e2cf60e81 in 1.0050 sec 2023-10-02T12:36:51,551 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:36:51,553 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:36:51,557 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:36:51,557 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:36:51,562 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:36:52,034 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-10-02T12:36:52,035 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16962502037113' 2023-10-02T12:36:52,036 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16962502037112' 2023-10-02T12:36:52,036 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16962502037111' 2023-10-02T12:36:52,042 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:36:52,074 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:36:52,082 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:36:52,097 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:36:52,097 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-10-02T12:36:52,100 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.0870 sec 2023-10-02T12:36:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:36:52,152 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(115): Backup backup_1696250209848 started at 1696250212152. 2023-10-02T12:36:52,197 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(119): Backup session backup_1696250209848 has been started. 2023-10-02T12:36:52,197 ERROR [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(228): Unexpected BackupException : Failed stage stage_0 in testing java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:188) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.runBackupAndFailAtStage(TestFullBackupWithFailures.java:67) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.testFullBackupWithFailures(TestFullBackupWithFailures.java:55) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:36:52,198 ERROR [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(240): BackupId=backup_1696250209848,startts=1696250212152,failedts=1696250212198,failedphase=REQUEST,failedmessage=Failed stage stage_0 in testing 2023-10-02T12:36:52,198 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:36:52,208 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-10-02T12:36:52,253 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable backup:system 2023-10-02T12:36:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-10-02T12:36:52,296 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250212296"}]},"ts":"1696250212296"} 2023-10-02T12:36:52,308 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-10-02T12:36:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:36:52,314 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-10-02T12:36:52,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, UNASSIGN}] 2023-10-02T12:36:52,323 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, UNASSIGN 2023-10-02T12:36:52,324 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:52,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE; CloseRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566}] 2023-10-02T12:36:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:36:52,497 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(119): Close 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:52,497 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1668): Closing 232c4115fe6f97bf9747948e2cf60e81, disabling compactions & flushes 2023-10-02T12:36:52,497 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1690): Closing region backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:52,497 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:52,498 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. after waiting 0 ms 2023-10-02T12:36:52,498 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:52,498 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(2819): Flushing 232c4115fe6f97bf9747948e2cf60e81 2/2 column families, dataSize=214 B heapSize=800 B 2023-10-02T12:36:52,551 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/89c6d3263e1745199d6f53705f873dee is 218, key is session:backup_1696250209848/session:context/1696250212190/Put/seqid=0 2023-10-02T12:36:52,611 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/89c6d3263e1745199d6f53705f873dee 2023-10-02T12:36:52,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:36:52,644 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/89c6d3263e1745199d6f53705f873dee as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/89c6d3263e1745199d6f53705f873dee 2023-10-02T12:36:52,661 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/89c6d3263e1745199d6f53705f873dee, entries=1, sequenceid=9, filesize=5.2 K 2023-10-02T12:36:52,682 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(3022): Finished flush of dataSize ~214 B/214, heapSize ~528 B/528, currentSize=0 B/0 for 232c4115fe6f97bf9747948e2cf60e81 in 184ms, sequenceid=9, compaction requested=false 2023-10-02T12:36:52,751 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2023-10-02T12:36:52,766 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:36:52,767 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1909): Closed backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:52,767 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1622): Region close journal for 232c4115fe6f97bf9747948e2cf60e81: 2023-10-02T12:36:52,774 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(163): Closed 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:52,774 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=35 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=CLOSED 2023-10-02T12:36:52,781 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=36, resume processing ppid=35 2023-10-02T12:36:52,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, ppid=35, state=SUCCESS; CloseRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566 in 447 msec 2023-10-02T12:36:52,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-10-02T12:36:52,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, UNASSIGN in 463 msec 2023-10-02T12:36:52,795 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250212795"}]},"ts":"1696250212795"} 2023-10-02T12:36:52,798 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-10-02T12:36:52,804 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-10-02T12:36:52,807 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; DisableTableProcedure table=backup:system in 549 msec 2023-10-02T12:36:52,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:36:52,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1696250212965 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:36:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250212967 (current time:1696250212967). 2023-10-02T12:36:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:36:52,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1696250212965 VERSION not specified, setting to 2 2023-10-02T12:36:53,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cf5d264, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:36:53,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:36:53,054 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53720, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:36:53,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:36:53,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:36:53,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=37, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250212965 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:36:53,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250212965 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-10-02T12:36:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-10-02T12:36:53,079 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250212965 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:36:53,081 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250212965 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:36:53,092 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot, current working dir path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696250212965 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:36:53,102 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250212965 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:36:53,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-10-02T12:36:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-10-02T12:36:53,550 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250212965 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-10-02T12:36:53,562 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-10-02T12:36:53,562 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-10-02T12:36:53,562 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:53,569 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(327): No files under family: meta 2023-10-02T12:36:53,569 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:53,583 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/89c6d3263e1745199d6f53705f873dee] hfiles 2023-10-02T12:36:53,584 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646 2023-10-02T12:36:53,584 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/89c6d3263e1745199d6f53705f873dee 2023-10-02T12:36:53,662 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-10-02T12:36:53,666 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250212965 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:36:53,670 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250212965 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:36:53,670 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1696250212965 2023-10-02T12:36:53,672 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696250212965 2023-10-02T12:36:53,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-10-02T12:36:53,793 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250212965 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:36:53,836 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250212965 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:36:53,838 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696250212965 to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1696250212965 2023-10-02T12:36:53,846 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=37, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250212965 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:36:53,847 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250212965 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 37 2023-10-02T12:36:53,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=37, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250212965 table=backup:system type=FLUSH ttl=0 } in 790 msec 2023-10-02T12:36:54,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=37 2023-10-02T12:36:54,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=38, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696250209953 type: FLUSH version: 2 ttl: 0 ) 2023-10-02T12:36:54,276 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-10-02T12:36:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:36:54,391 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2023-10-02T12:36:54,391 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-10-02T12:36:54,399 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:36:54,416 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696250209953 type: FLUSH version: 2 ttl: 0 2023-10-02T12:36:54,417 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system 2023-10-02T12:36:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:36:54,436 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-10-02T12:36:54,436 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:54,523 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:54,574 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/89c6d3263e1745199d6f53705f873dee to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/archive/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/89c6d3263e1745199d6f53705f873dee 2023-10-02T12:36:54,575 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:54,579 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696250209953 type: FLUSH version: 2 ttl: 0 2023-10-02T12:36:54,599 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696250214599"}]},"ts":"9223372036854775807"} 2023-10-02T12:36:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:36:54,742 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:36:54,742 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:36:54,744 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1696250214600"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250214600"}]},"ts":"1696250214600"} 2023-10-02T12:36:54,756 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:36:54,756 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-10-02T12:36:54,756 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:36:54,757 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-10-02T12:36:54,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696250209953 type: FLUSH version: 2 ttl: 0 ) in 521 msec 2023-10-02T12:36:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:36:54,940 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1696250212965" 2023-10-02T12:36:54,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1696250212965 2023-10-02T12:36:54,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster$11(2631): Client=jenkins//172.31.12.81 enable backup:system 2023-10-02T12:36:54,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=39, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-10-02T12:36:55,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-10-02T12:36:55,023 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-10-02T12:36:55,024 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250215023"}]},"ts":"1696250215023"} 2023-10-02T12:36:55,027 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-10-02T12:36:55,046 DEBUG [PEWorker-2 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-10-02T12:36:55,046 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN}] 2023-10-02T12:36:55,057 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN 2023-10-02T12:36:55,076 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=40, ppid=39, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-10-02T12:36:55,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-10-02T12:36:55,230 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:55,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE; OpenRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566}] 2023-10-02T12:36:55,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-10-02T12:36:55,419 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(130): Open backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:55,419 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7254): Opening region: {ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:36:55,419 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:36:55,420 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:55,420 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(888): Instantiated backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:36:55,420 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7296): checking encryption for 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:55,420 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7299): checking classloading for 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:55,433 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:55,443 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 232c4115fe6f97bf9747948e2cf60e81 columnFamilyName meta 2023-10-02T12:36:55,443 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:55,451 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(324): Store=232c4115fe6f97bf9747948e2cf60e81/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:36:55,454 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:55,456 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 232c4115fe6f97bf9747948e2cf60e81 columnFamilyName session 2023-10-02T12:36:55,456 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:36:55,515 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646 2023-10-02T12:36:55,515 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(324): Store=232c4115fe6f97bf9747948e2cf60e81/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:36:55,518 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:55,528 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:55,531 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:36:55,538 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1076): writing seq id for 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:55,543 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): Opened 232c4115fe6f97bf9747948e2cf60e81; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9572968000, jitterRate=-0.10844787955284119}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:36:55,544 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(992): Region open journal for 232c4115fe6f97bf9747948e2cf60e81: 2023-10-02T12:36:55,546 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81., pid=41, masterSystemTime=1696250215402 2023-10-02T12:36:55,555 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:55,555 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(158): Opened backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:55,556 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=40 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=OPEN, openSeqNum=13, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:55,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=41, resume processing ppid=40 2023-10-02T12:36:55,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, ppid=40, state=SUCCESS; OpenRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566 in 321 msec 2023-10-02T12:36:55,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=40, resume processing ppid=39 2023-10-02T12:36:55,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, ppid=39, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN in 521 msec 2023-10-02T12:36:55,570 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250215570"}]},"ts":"1696250215570"} 2023-10-02T12:36:55,572 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-10-02T12:36:55,580 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-10-02T12:36:55,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, state=SUCCESS; EnableTableProcedure table=backup:system in 611 msec 2023-10-02T12:36:55,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=39 2023-10-02T12:36:55,629 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-10-02T12:36:55,629 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:36:55,645 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:36:55,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:36:55,654 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:36:55,654 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-10-02T12:36:55,672 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:36:55,735 ERROR [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(246): Backup backup_1696250209848 failed. Backup session finished. Status: FAILURE 2023-10-02T12:36:55,735 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:36:55,735 ERROR [Listener at jenkins-hbase3.apache.org/34907 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.runBackupAndFailAtStage(TestFullBackupWithFailures.java:67) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.testFullBackupWithFailures(TestFullBackupWithFailures.java:55) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_0 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:188) ~[test-classes/:?] ... 32 more 2023-10-02T12:36:55,822 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] backup.TestFullBackupWithFailures(54): Running stage 1 2023-10-02T12:36:55,893 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14eb77c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:36:55,904 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:36:55,907 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37412, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:36:55,922 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:36:55,925 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53732, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:36:56,103 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:36:56,119 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:36:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250216119 (current time:1696250216119). 2023-10-02T12:36:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:36:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:36:56,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60d52f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:36:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:36:56,193 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53748, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:36:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:36:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:36:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=42, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:36:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-10-02T12:36:56,200 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:36:56,202 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:36:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:36:56,206 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot, current working dir path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:36:56,210 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:36:56,244 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:36:56,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure 232c4115fe6f97bf9747948e2cf60e81}] 2023-10-02T12:36:56,247 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:36:56,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37985 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=43 2023-10-02T12:36:56,400 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:56,402 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2819): Flushing 232c4115fe6f97bf9747948e2cf60e81 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-10-02T12:36:56,425 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/670bddd78f654d84b6033b3d735ecaee is 256, key is session:backup_1696250209848/session:context/1696250215658/Put/seqid=0 2023-10-02T12:36:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:36:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:36:56,859 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/670bddd78f654d84b6033b3d735ecaee 2023-10-02T12:36:56,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/670bddd78f654d84b6033b3d735ecaee as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/670bddd78f654d84b6033b3d735ecaee 2023-10-02T12:36:56,877 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/670bddd78f654d84b6033b3d735ecaee, entries=2, sequenceid=18, filesize=5.2 K 2023-10-02T12:36:56,880 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for 232c4115fe6f97bf9747948e2cf60e81 in 478ms, sequenceid=18, compaction requested=false 2023-10-02T12:36:56,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.HRegion(2520): Flush status journal for 232c4115fe6f97bf9747948e2cf60e81: 2023-10-02T12:36:56,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. for snapshot_backup_system completed. 2023-10-02T12:36:56,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:36:56,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:36:56,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:36:56,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/670bddd78f654d84b6033b3d735ecaee] hfiles 2023-10-02T12:36:56,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646 for snapshot=snapshot_backup_system 2023-10-02T12:36:56,880 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/670bddd78f654d84b6033b3d735ecaee for snapshot=snapshot_backup_system 2023-10-02T12:36:56,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:56,893 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=43 2023-10-02T12:36:56,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.HMaster(3995): Remote procedure done, pid=43 2023-10-02T12:36:56,894 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:56,895 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=43, ppid=42, state=RUNNABLE; SnapshotRegionProcedure 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:56,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=43, resume processing ppid=42 2023-10-02T12:36:56,898 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:36:56,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, ppid=42, state=SUCCESS; SnapshotRegionProcedure 232c4115fe6f97bf9747948e2cf60e81 in 651 msec 2023-10-02T12:36:56,898 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:36:56,899 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:36:56,899 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:36:56,900 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:36:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:36:57,336 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:36:57,355 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:36:57,356 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:36:57,359 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=42, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:36:57,359 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 42 2023-10-02T12:36:57,361 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=42, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.1630 sec 2023-10-02T12:36:58,271 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:36:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:36:58,336 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(115): Backup backup_1696250216018 started at 1696250218334. 2023-10-02T12:36:58,338 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(119): Backup session backup_1696250216018 has been started. 2023-10-02T12:36:58,342 ERROR [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(228): Unexpected BackupException : Failed stage stage_1 in testing java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:199) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.runBackupAndFailAtStage(TestFullBackupWithFailures.java:67) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.testFullBackupWithFailures(TestFullBackupWithFailures.java:55) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:36:58,343 ERROR [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(240): BackupId=backup_1696250216018,startts=1696250218334,failedts=1696250218343,failedphase=REQUEST,failedmessage=Failed stage stage_1 in testing 2023-10-02T12:36:58,343 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:36:58,344 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-10-02T12:36:58,351 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable backup:system 2023-10-02T12:36:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=44, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-10-02T12:36:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:36:58,359 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250218359"}]},"ts":"1696250218359"} 2023-10-02T12:36:58,363 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-10-02T12:36:58,365 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-10-02T12:36:58,366 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, UNASSIGN}] 2023-10-02T12:36:58,370 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, UNASSIGN 2023-10-02T12:36:58,371 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:36:58,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; CloseRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566}] 2023-10-02T12:36:58,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:36:58,527 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(119): Close 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:58,527 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1668): Closing 232c4115fe6f97bf9747948e2cf60e81, disabling compactions & flushes 2023-10-02T12:36:58,527 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1690): Closing region backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:58,527 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:58,527 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. after waiting 0 ms 2023-10-02T12:36:58,527 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:58,527 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(2819): Flushing 232c4115fe6f97bf9747948e2cf60e81 2/2 column families, dataSize=289 B heapSize=952 B 2023-10-02T12:36:58,543 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/meta/0970a115311f4d4ca471c6a43ce06ff0 is 79, key is startcode:hdfs://localhost:40817/backupUT/meta:startcode/1696250218341/Put/seqid=0 2023-10-02T12:36:58,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:36:58,957 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=75 B at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/meta/0970a115311f4d4ca471c6a43ce06ff0 2023-10-02T12:36:58,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:36:58,972 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/9bd3c7b5af39436caefe5768b1616f79 is 218, key is session:backup_1696250216018/session:context/1696250218337/Put/seqid=0 2023-10-02T12:36:59,199 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-10-02T12:36:59,199 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-10-02T12:36:59,402 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/9bd3c7b5af39436caefe5768b1616f79 2023-10-02T12:36:59,415 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/meta/0970a115311f4d4ca471c6a43ce06ff0 as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/0970a115311f4d4ca471c6a43ce06ff0 2023-10-02T12:36:59,422 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/0970a115311f4d4ca471c6a43ce06ff0, entries=1, sequenceid=23, filesize=5.1 K 2023-10-02T12:36:59,423 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/9bd3c7b5af39436caefe5768b1616f79 as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/9bd3c7b5af39436caefe5768b1616f79 2023-10-02T12:36:59,438 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/9bd3c7b5af39436caefe5768b1616f79, entries=1, sequenceid=23, filesize=5.2 K 2023-10-02T12:36:59,439 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(3022): Finished flush of dataSize ~289 B/289, heapSize ~920 B/920, currentSize=0 B/0 for 232c4115fe6f97bf9747948e2cf60e81 in 912ms, sequenceid=23, compaction requested=true 2023-10-02T12:36:59,453 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=12 2023-10-02T12:36:59,454 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:36:59,455 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1909): Closed backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:36:59,455 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1622): Region close journal for 232c4115fe6f97bf9747948e2cf60e81: 2023-10-02T12:36:59,458 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=CLOSED 2023-10-02T12:36:59,460 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(163): Closed 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:36:59,463 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=46, resume processing ppid=45 2023-10-02T12:36:59,463 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, ppid=45, state=SUCCESS; CloseRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566 in 1.0870 sec 2023-10-02T12:36:59,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=45, resume processing ppid=44 2023-10-02T12:36:59,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, ppid=44, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, UNASSIGN in 1.0970 sec 2023-10-02T12:36:59,466 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250219466"}]},"ts":"1696250219466"} 2023-10-02T12:36:59,468 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-10-02T12:36:59,470 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-10-02T12:36:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:36:59,478 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, state=SUCCESS; DisableTableProcedure table=backup:system in 1.1260 sec 2023-10-02T12:37:00,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:37:00,494 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1696250220491 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:37:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250220494 (current time:1696250220494). 2023-10-02T12:37:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:37:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1696250220491 VERSION not specified, setting to 2 2023-10-02T12:37:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12e308f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:37:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:37:00,513 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53082, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:37:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:37:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=47, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250220491 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:37:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250220491 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2023-10-02T12:37:00,519 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250220491 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:37:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-10-02T12:37:00,520 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250220491 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:37:00,524 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot, current working dir path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696250220491 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:37:00,526 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250220491 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:37:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-10-02T12:37:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-10-02T12:37:00,940 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250220491 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-10-02T12:37:00,942 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-10-02T12:37:00,942 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-10-02T12:37:00,943 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:00,944 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/0970a115311f4d4ca471c6a43ce06ff0] hfiles 2023-10-02T12:37:00,944 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/0970a115311f4d4ca471c6a43ce06ff0 2023-10-02T12:37:00,945 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:00,947 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/670bddd78f654d84b6033b3d735ecaee, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/9bd3c7b5af39436caefe5768b1616f79] hfiles 2023-10-02T12:37:00,947 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646 2023-10-02T12:37:00,948 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/670bddd78f654d84b6033b3d735ecaee 2023-10-02T12:37:00,948 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/9bd3c7b5af39436caefe5768b1616f79 2023-10-02T12:37:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-10-02T12:37:01,367 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-10-02T12:37:01,369 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250220491 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:37:01,370 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250220491 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:37:01,370 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1696250220491 2023-10-02T12:37:01,371 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696250220491 2023-10-02T12:37:01,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-10-02T12:37:01,794 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250220491 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:37:01,808 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250220491 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:37:01,809 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696250220491 to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1696250220491 2023-10-02T12:37:01,811 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=47, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250220491 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:37:01,811 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250220491 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 47 2023-10-02T12:37:01,812 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=47, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250220491 table=backup:system type=FLUSH ttl=0 } in 1.2950 sec 2023-10-02T12:37:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-10-02T12:37:02,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=48, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696250216119 type: FLUSH version: 2 ttl: 0 ) 2023-10-02T12:37:02,661 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36327 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-10-02T12:37:02,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-10-02T12:37:02,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-10-02T12:37:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-10-02T12:37:03,096 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-10-02T12:37:03,096 INFO [PEWorker-3 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-10-02T12:37:03,098 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:37:03,101 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696250216119 type: FLUSH version: 2 ttl: 0 2023-10-02T12:37:03,101 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system 2023-10-02T12:37:03,105 DEBUG [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-10-02T12:37:03,105 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:03,114 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/0970a115311f4d4ca471c6a43ce06ff0 to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/archive/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/0970a115311f4d4ca471c6a43ce06ff0 2023-10-02T12:37:03,115 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:03,119 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/9bd3c7b5af39436caefe5768b1616f79 to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/archive/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/9bd3c7b5af39436caefe5768b1616f79 2023-10-02T12:37:03,119 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:03,119 INFO [PEWorker-3 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696250216119 type: FLUSH version: 2 ttl: 0 2023-10-02T12:37:03,121 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696250223120"}]},"ts":"9223372036854775807"} 2023-10-02T12:37:03,124 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:37:03,124 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:37:03,124 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1696250223121"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250223121"}]},"ts":"1696250223121"} 2023-10-02T12:37:03,126 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:37:03,126 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-10-02T12:37:03,126 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:37:03,126 INFO [PEWorker-3 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-10-02T12:37:03,130 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696250216119 type: FLUSH version: 2 ttl: 0 ) in 476 msec 2023-10-02T12:37:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=48 2023-10-02T12:37:03,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1696250220491" 2023-10-02T12:37:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1696250220491 2023-10-02T12:37:03,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster$11(2631): Client=jenkins//172.31.12.81 enable backup:system 2023-10-02T12:37:03,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=49, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-10-02T12:37:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-10-02T12:37:03,293 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-10-02T12:37:03,294 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250223294"}]},"ts":"1696250223294"} 2023-10-02T12:37:03,296 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-10-02T12:37:03,300 DEBUG [PEWorker-2 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-10-02T12:37:03,300 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN}] 2023-10-02T12:37:03,302 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN 2023-10-02T12:37:03,303 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-10-02T12:37:03,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-10-02T12:37:03,455 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=50 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:03,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; OpenRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566}] 2023-10-02T12:37:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-10-02T12:37:03,614 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(130): Open backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:03,614 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7254): Opening region: {ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:37:03,615 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:37:03,615 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:03,615 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(888): Instantiated backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:37:03,615 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7296): checking encryption for 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:03,615 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7299): checking classloading for 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:03,619 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:03,620 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 232c4115fe6f97bf9747948e2cf60e81 columnFamilyName meta 2023-10-02T12:37:03,620 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:03,622 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(324): Store=232c4115fe6f97bf9747948e2cf60e81/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:37:03,622 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:03,624 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 232c4115fe6f97bf9747948e2cf60e81 columnFamilyName session 2023-10-02T12:37:03,624 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:03,635 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646 2023-10-02T12:37:03,643 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/670bddd78f654d84b6033b3d735ecaee 2023-10-02T12:37:03,643 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(324): Store=232c4115fe6f97bf9747948e2cf60e81/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:37:03,644 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:03,646 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:03,648 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:37:03,649 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1076): writing seq id for 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:03,651 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): Opened 232c4115fe6f97bf9747948e2cf60e81; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10996319840, jitterRate=0.024112090468406677}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:37:03,651 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(992): Region open journal for 232c4115fe6f97bf9747948e2cf60e81: 2023-10-02T12:37:03,652 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81., pid=51, masterSystemTime=1696250223609 2023-10-02T12:37:03,654 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:03,655 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(158): Opened backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:03,655 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=50 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=OPEN, openSeqNum=27, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:03,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-10-02T12:37:03,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; OpenRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566 in 200 msec 2023-10-02T12:37:03,660 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=50, resume processing ppid=49 2023-10-02T12:37:03,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, ppid=49, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN in 359 msec 2023-10-02T12:37:03,662 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250223661"}]},"ts":"1696250223661"} 2023-10-02T12:37:03,663 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-10-02T12:37:03,666 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-10-02T12:37:03,668 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, state=SUCCESS; EnableTableProcedure table=backup:system in 379 msec 2023-10-02T12:37:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-10-02T12:37:03,897 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-10-02T12:37:03,898 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:37:03,909 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:37:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:37:03,915 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:37:03,915 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-10-02T12:37:03,918 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:37:03,924 ERROR [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(246): Backup backup_1696250216018 failed. Backup session finished. Status: FAILURE 2023-10-02T12:37:03,925 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:03,925 ERROR [Listener at jenkins-hbase3.apache.org/34907 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.runBackupAndFailAtStage(TestFullBackupWithFailures.java:67) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.testFullBackupWithFailures(TestFullBackupWithFailures.java:55) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_1 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:199) ~[test-classes/:?] ... 32 more 2023-10-02T12:37:03,969 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] backup.TestFullBackupWithFailures(54): Running stage 2 2023-10-02T12:37:03,992 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6bb66e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:37:03,993 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:37:03,995 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45862, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:37:03,999 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:37:04,000 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53096, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:37:04,049 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:37:04,053 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:37:04,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250224053 (current time:1696250224053). 2023-10-02T12:37:04,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:37:04,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:37:04,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@505cbf9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:37:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:37:04,064 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53102, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:37:04,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:04,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:37:04,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:37:04,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-10-02T12:37:04,068 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:37:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:37:04,069 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:37:04,071 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot, current working dir path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:37:04,074 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:37:04,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:37:04,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:37:04,480 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:37:04,481 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 232c4115fe6f97bf9747948e2cf60e81}] 2023-10-02T12:37:04,483 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:04,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37985 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2023-10-02T12:37:04,635 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:04,636 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2819): Flushing 232c4115fe6f97bf9747948e2cf60e81 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-10-02T12:37:04,651 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/dcd80847668348169ebbbaf8dcaff038 is 256, key is session:backup_1696250216018/session:context/1696250223916/Put/seqid=0 2023-10-02T12:37:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:37:04,958 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:37:05,056 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/dcd80847668348169ebbbaf8dcaff038 2023-10-02T12:37:05,066 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/dcd80847668348169ebbbaf8dcaff038 as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/dcd80847668348169ebbbaf8dcaff038 2023-10-02T12:37:05,073 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/dcd80847668348169ebbbaf8dcaff038, entries=2, sequenceid=32, filesize=5.2 K 2023-10-02T12:37:05,074 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for 232c4115fe6f97bf9747948e2cf60e81 in 438ms, sequenceid=32, compaction requested=true 2023-10-02T12:37:05,074 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2520): Flush status journal for 232c4115fe6f97bf9747948e2cf60e81: 2023-10-02T12:37:05,074 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. for snapshot_backup_system completed. 2023-10-02T12:37:05,074 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:37:05,074 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:37:05,074 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:37:05,074 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/670bddd78f654d84b6033b3d735ecaee, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/dcd80847668348169ebbbaf8dcaff038] hfiles 2023-10-02T12:37:05,074 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646 for snapshot=snapshot_backup_system 2023-10-02T12:37:05,074 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/670bddd78f654d84b6033b3d735ecaee for snapshot=snapshot_backup_system 2023-10-02T12:37:05,074 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/dcd80847668348169ebbbaf8dcaff038 for snapshot=snapshot_backup_system 2023-10-02T12:37:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:37:05,481 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:05,481 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=53 2023-10-02T12:37:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster(3995): Remote procedure done, pid=53 2023-10-02T12:37:05,482 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:05,483 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:05,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=53, resume processing ppid=52 2023-10-02T12:37:05,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, ppid=52, state=SUCCESS; SnapshotRegionProcedure 232c4115fe6f97bf9747948e2cf60e81 in 1.0030 sec 2023-10-02T12:37:05,486 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:37:05,487 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:37:05,488 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:37:05,488 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:37:05,488 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:37:05,903 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:37:05,910 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:37:05,911 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:37:05,912 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:37:05,912 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-10-02T12:37:05,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8460 sec 2023-10-02T12:37:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:37:06,176 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(115): Backup backup_1696250224034 started at 1696250226176. 2023-10-02T12:37:06,178 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(119): Backup session backup_1696250224034 has been started. 2023-10-02T12:37:06,181 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] backup.TestBackupBase$FullTableBackupClientForTest(205): Execute roll log procedure for full backup ... 2023-10-02T12:37:06,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:37:06,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:37:06,203 INFO [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:37:06,203 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:37:06,204 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:37:06,204 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:37:06,204 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:37:06,206 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:06,206 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:37:06,206 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:37:06,207 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:37:06,207 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:37:06,208 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:37:06,208 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:37:06,208 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:37:06,210 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:37:06,210 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:37:06,210 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:37:06,210 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:37:06,211 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:37:06,211 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:37:06,211 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,37985,1696250191566' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:37:06,214 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:37:06,214 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:06,214 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:37:06,214 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:37:06,214 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:37:06,216 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:37:06,216 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:37:06,217 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:37:06,217 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:06,217 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:37:06,218 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,37985,1696250191566' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:37:06,218 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:37:06,218 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@5b02891e[Count = 0] remaining members to acquire global barrier 2023-10-02T12:37:06,218 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:37:06,220 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:37:06,220 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:37:06,221 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:37:06,221 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:37:06,222 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:37:06,222 DEBUG [rs(jenkins-hbase3.apache.org,37985,1696250191566)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:06,222 INFO [rs(jenkins-hbase3.apache.org,37985,1696250191566)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696250193662 highest: 1696250193662 on jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:06,222 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37985%2C1696250191566.meta:.meta(num 1696250194253) roll requested 2023-10-02T12:37:06,235 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.meta.1696250226222.meta, exclude list is [], retry=0 2023-10-02T12:37:06,240 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33097,DS-0c137cf7-1875-4349-b1e4-5237971c808a,DISK] 2023-10-02T12:37:06,244 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.meta.1696250194253.meta with entries=56, filesize=15.96 KB; new WAL /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.meta.1696250226222.meta 2023-10-02T12:37:06,245 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33097,DS-0c137cf7-1875-4349-b1e4-5237971c808a,DISK]] 2023-10-02T12:37:06,246 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.meta.1696250194253.meta is not closed yet, will try archiving it next time 2023-10-02T12:37:06,246 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37985%2C1696250191566:(num 1696250193662) roll requested 2023-10-02T12:37:06,261 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.1696250226246, exclude list is [], retry=0 2023-10-02T12:37:06,263 DEBUG [rs(jenkins-hbase3.apache.org,37985,1696250191566)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-10-02T12:37:06,263 INFO [rs(jenkins-hbase3.apache.org,37985,1696250191566)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696250226246 on jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:06,263 DEBUG [rs(jenkins-hbase3.apache.org,37985,1696250191566)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:37:06,266 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33097,DS-0c137cf7-1875-4349-b1e4-5237971c808a,DISK] 2023-10-02T12:37:06,266 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45868, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-10-02T12:37:06,269 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.1696250193662 with entries=40, filesize=9.29 KB; new WAL /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.1696250226246 2023-10-02T12:37:06,269 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33097,DS-0c137cf7-1875-4349-b1e4-5237971c808a,DISK]] 2023-10-02T12:37:06,270 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.1696250193662 is not closed yet, will try archiving it next time 2023-10-02T12:37:06,272 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.1696250193662 not finished, retry = 0 2023-10-02T12:37:06,290 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:37:06,290 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,37985,1696250191566' in zk 2023-10-02T12:37:06,294 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:06,294 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:37:06,294 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:37:06,294 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:37:06,294 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:37:06,294 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:37:06,296 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:37:06,296 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:37:06,297 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:37:06,297 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:06,298 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:37:06,298 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:37:06,298 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:06,299 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,37985,1696250191566': 2023-10-02T12:37:06,300 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,37985,1696250191566' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:37:06,300 INFO [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:37:06,300 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:37:06,300 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:37:06,300 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:37:06,300 INFO [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:37:06,305 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:37:06,305 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:37:06,305 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:37:06,305 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:37:06,305 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:37:06,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:37:06,306 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:37:06,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:37:06,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:37:06,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:37:06,307 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:37:06,307 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:37:06,310 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:06,311 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:37:06,312 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:37:06,313 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:06,322 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:37:06,322 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:37:06,325 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:37:06,325 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:37:06,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:37:06,326 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:37:06,326 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:37:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:37:06,437 ERROR [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(228): Unexpected BackupException : Failed stage stage_2 in testing java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:211) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.runBackupAndFailAtStage(TestFullBackupWithFailures.java:67) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.testFullBackupWithFailures(TestFullBackupWithFailures.java:55) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:37:06,438 ERROR [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(240): BackupId=backup_1696250224034,startts=1696250226176,failedts=1696250226438,failedphase=REQUEST,failedmessage=Failed stage stage_2 in testing 2023-10-02T12:37:06,438 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:37:06,439 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-10-02T12:37:06,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable backup:system 2023-10-02T12:37:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-10-02T12:37:06,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-10-02T12:37:06,451 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250226451"}]},"ts":"1696250226451"} 2023-10-02T12:37:06,453 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-10-02T12:37:06,455 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-10-02T12:37:06,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, UNASSIGN}] 2023-10-02T12:37:06,458 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, UNASSIGN 2023-10-02T12:37:06,459 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:06,461 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; CloseRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566}] 2023-10-02T12:37:06,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-10-02T12:37:06,614 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(119): Close 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:06,614 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1668): Closing 232c4115fe6f97bf9747948e2cf60e81, disabling compactions & flushes 2023-10-02T12:37:06,614 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1690): Closing region backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:06,614 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:06,614 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. after waiting 0 ms 2023-10-02T12:37:06,614 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:06,614 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(2819): Flushing 232c4115fe6f97bf9747948e2cf60e81 2/2 column families, dataSize=401 B heapSize=1.11 KB 2023-10-02T12:37:06,635 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/meta/2620d90fe9ae462eabf6f4b4ae73916a is 116, key is rslogts:hdfs://localhost:40817/backupUT\x00jenkins-hbase3.apache.org:37985/meta:rs-log-ts/1696250226288/Put/seqid=0 2023-10-02T12:37:06,641 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=187 B at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/meta/2620d90fe9ae462eabf6f4b4ae73916a 2023-10-02T12:37:06,654 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/04c4e5dc3cb848f6b6eb2398928cd5cc is 218, key is session:backup_1696250224034/session:context/1696250226177/Put/seqid=0 2023-10-02T12:37:06,660 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/04c4e5dc3cb848f6b6eb2398928cd5cc 2023-10-02T12:37:06,668 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/meta/2620d90fe9ae462eabf6f4b4ae73916a as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/2620d90fe9ae462eabf6f4b4ae73916a 2023-10-02T12:37:06,676 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/2620d90fe9ae462eabf6f4b4ae73916a, entries=2, sequenceid=38, filesize=5.3 K 2023-10-02T12:37:06,677 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/04c4e5dc3cb848f6b6eb2398928cd5cc as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/04c4e5dc3cb848f6b6eb2398928cd5cc 2023-10-02T12:37:06,688 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/04c4e5dc3cb848f6b6eb2398928cd5cc, entries=1, sequenceid=38, filesize=5.2 K 2023-10-02T12:37:06,689 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(3022): Finished flush of dataSize ~401 B/401, heapSize ~1.08 KB/1104, currentSize=0 B/0 for 232c4115fe6f97bf9747948e2cf60e81 in 75ms, sequenceid=38, compaction requested=true 2023-10-02T12:37:06,705 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/recovered.edits/41.seqid, newMaxSeqId=41, maxSeqId=26 2023-10-02T12:37:06,705 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:37:06,707 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1909): Closed backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:06,707 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1622): Region close journal for 232c4115fe6f97bf9747948e2cf60e81: 2023-10-02T12:37:06,709 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(163): Closed 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:06,709 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=CLOSED 2023-10-02T12:37:06,713 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=56, resume processing ppid=55 2023-10-02T12:37:06,713 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=56, ppid=55, state=SUCCESS; CloseRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566 in 250 msec 2023-10-02T12:37:06,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=55, resume processing ppid=54 2023-10-02T12:37:06,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, UNASSIGN in 257 msec 2023-10-02T12:37:06,716 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250226715"}]},"ts":"1696250226715"} 2023-10-02T12:37:06,719 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-10-02T12:37:06,721 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-10-02T12:37:06,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=54, state=SUCCESS; DisableTableProcedure table=backup:system in 276 msec 2023-10-02T12:37:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-10-02T12:37:06,764 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1696250226763 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:37:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250226764 (current time:1696250226764). 2023-10-02T12:37:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:37:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1696250226763 VERSION not specified, setting to 2 2023-10-02T12:37:06,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4906c7be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:37:06,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:37:06,775 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53112, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:37:06,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:06,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:37:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=57, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250226763 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:37:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250226763 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 57 2023-10-02T12:37:06,779 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250226763 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:37:06,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-10-02T12:37:06,779 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250226763 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:37:06,782 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot, current working dir path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696250226763 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:37:06,784 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250226763 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:37:06,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-10-02T12:37:07,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-10-02T12:37:07,193 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250226763 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-10-02T12:37:07,194 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-10-02T12:37:07,194 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-10-02T12:37:07,194 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:07,195 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/2620d90fe9ae462eabf6f4b4ae73916a] hfiles 2023-10-02T12:37:07,196 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/2620d90fe9ae462eabf6f4b4ae73916a 2023-10-02T12:37:07,196 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:07,198 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/04c4e5dc3cb848f6b6eb2398928cd5cc, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/670bddd78f654d84b6033b3d735ecaee, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/dcd80847668348169ebbbaf8dcaff038] hfiles 2023-10-02T12:37:07,198 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/4): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/04c4e5dc3cb848f6b6eb2398928cd5cc 2023-10-02T12:37:07,198 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/4): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646 2023-10-02T12:37:07,198 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/4): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/670bddd78f654d84b6033b3d735ecaee 2023-10-02T12:37:07,198 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (4/4): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/dcd80847668348169ebbbaf8dcaff038 2023-10-02T12:37:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-10-02T12:37:07,605 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-10-02T12:37:07,606 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250226763 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:37:07,607 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250226763 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:37:07,607 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1696250226763 2023-10-02T12:37:07,608 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696250226763 2023-10-02T12:37:07,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-10-02T12:37:08,020 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250226763 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:37:08,033 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250226763 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:37:08,034 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696250226763 to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1696250226763 2023-10-02T12:37:08,036 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=57, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250226763 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:37:08,036 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250226763 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 57 2023-10-02T12:37:08,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=57, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=57, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250226763 table=backup:system type=FLUSH ttl=0 } in 1.2600 sec 2023-10-02T12:37:08,217 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-10-02T12:37:08,221 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45880, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-10-02T12:37:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-10-02T12:37:08,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=58, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696250224053 type: FLUSH version: 2 ttl: 0 ) 2023-10-02T12:37:08,897 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-10-02T12:37:08,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-10-02T12:37:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-10-02T12:37:09,199 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-10-02T12:37:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-10-02T12:37:09,312 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-10-02T12:37:09,312 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-10-02T12:37:09,313 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:37:09,316 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696250224053 type: FLUSH version: 2 ttl: 0 2023-10-02T12:37:09,316 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system 2023-10-02T12:37:09,319 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-10-02T12:37:09,319 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:09,326 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/2620d90fe9ae462eabf6f4b4ae73916a to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/archive/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/2620d90fe9ae462eabf6f4b4ae73916a 2023-10-02T12:37:09,326 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:09,330 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/04c4e5dc3cb848f6b6eb2398928cd5cc to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/archive/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/04c4e5dc3cb848f6b6eb2398928cd5cc 2023-10-02T12:37:09,330 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:09,330 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696250224053 type: FLUSH version: 2 ttl: 0 2023-10-02T12:37:09,331 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696250229331"}]},"ts":"9223372036854775807"} 2023-10-02T12:37:09,334 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:37:09,334 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:37:09,334 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1696250229332"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250229332"}]},"ts":"1696250229332"} 2023-10-02T12:37:09,336 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:37:09,336 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-10-02T12:37:09,336 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:37:09,336 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-10-02T12:37:09,338 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=58, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696250224053 type: FLUSH version: 2 ttl: 0 ) in 441 msec 2023-10-02T12:37:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-10-02T12:37:09,505 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1696250226763" 2023-10-02T12:37:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1696250226763 2023-10-02T12:37:09,509 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster$11(2631): Client=jenkins//172.31.12.81 enable backup:system 2023-10-02T12:37:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=59, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-10-02T12:37:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=59 2023-10-02T12:37:09,514 INFO [PEWorker-3 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-10-02T12:37:09,514 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250229514"}]},"ts":"1696250229514"} 2023-10-02T12:37:09,515 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-10-02T12:37:09,518 DEBUG [PEWorker-3 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-10-02T12:37:09,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN}] 2023-10-02T12:37:09,520 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN 2023-10-02T12:37:09,521 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-10-02T12:37:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=59 2023-10-02T12:37:09,672 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=60 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:09,674 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE; OpenRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566}] 2023-10-02T12:37:09,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=59 2023-10-02T12:37:09,830 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(130): Open backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:09,830 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7254): Opening region: {ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:37:09,830 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:37:09,830 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:09,830 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(888): Instantiated backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:37:09,830 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7296): checking encryption for 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:09,831 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7299): checking classloading for 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:09,832 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:09,833 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 232c4115fe6f97bf9747948e2cf60e81 columnFamilyName meta 2023-10-02T12:37:09,833 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:09,833 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(324): Store=232c4115fe6f97bf9747948e2cf60e81/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:37:09,834 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:09,834 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 232c4115fe6f97bf9747948e2cf60e81 columnFamilyName session 2023-10-02T12:37:09,834 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:09,841 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646 2023-10-02T12:37:09,846 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/670bddd78f654d84b6033b3d735ecaee 2023-10-02T12:37:09,851 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/dcd80847668348169ebbbaf8dcaff038 2023-10-02T12:37:09,851 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(324): Store=232c4115fe6f97bf9747948e2cf60e81/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:37:09,852 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:09,853 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:09,855 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:37:09,856 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1076): writing seq id for 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:09,857 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): Opened 232c4115fe6f97bf9747948e2cf60e81; next sequenceid=42; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10626963040, jitterRate=-0.01028694212436676}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:37:09,858 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(992): Region open journal for 232c4115fe6f97bf9747948e2cf60e81: 2023-10-02T12:37:09,858 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81., pid=61, masterSystemTime=1696250229826 2023-10-02T12:37:09,861 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.CompactSplit(403): Add compact mark for store 232c4115fe6f97bf9747948e2cf60e81:session, priority=-2147483648, current under compaction store size is 1 2023-10-02T12:37:09,862 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-10-02T12:37:09,862 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-10-02T12:37:09,864 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:09,864 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(158): Opened backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:09,864 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=60 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=OPEN, openSeqNum=42, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:09,866 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 15714 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-10-02T12:37:09,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=61, resume processing ppid=60 2023-10-02T12:37:09,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=61, ppid=60, state=SUCCESS; OpenRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566 in 192 msec 2023-10-02T12:37:09,868 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] regionserver.HStore(1534): 232c4115fe6f97bf9747948e2cf60e81/session is initiating minor compaction (all files) 2023-10-02T12:37:09,868 INFO [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 232c4115fe6f97bf9747948e2cf60e81/session in backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:09,868 INFO [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/670bddd78f654d84b6033b3d735ecaee, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/dcd80847668348169ebbbaf8dcaff038] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp, totalSize=15.3 K 2023-10-02T12:37:09,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=60, resume processing ppid=59 2023-10-02T12:37:09,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=60, ppid=59, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN in 350 msec 2023-10-02T12:37:09,870 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] compactions.Compactor(223): Compacting 547c8c6158404697bd5ec227f6f3d646, keycount=1, bloomtype=ROW, size=4.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1696250209934 2023-10-02T12:37:09,870 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250229870"}]},"ts":"1696250229870"} 2023-10-02T12:37:09,870 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] compactions.Compactor(223): Compacting 670bddd78f654d84b6033b3d735ecaee, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1696250215658 2023-10-02T12:37:09,871 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] compactions.Compactor(223): Compacting dcd80847668348169ebbbaf8dcaff038, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=32, earliestPutTs=1696250223916 2023-10-02T12:37:09,872 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-10-02T12:37:09,873 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-10-02T12:37:09,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=59, state=SUCCESS; EnableTableProcedure table=backup:system in 364 msec 2023-10-02T12:37:09,898 INFO [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 232c4115fe6f97bf9747948e2cf60e81#session#compaction#8 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-10-02T12:37:09,898 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/346633fda9724350a60ab080f906d768 is 256, key is session:backup_1696250209848/session:context/1696250215658/Put/seqid=0 2023-10-02T12:37:09,910 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/346633fda9724350a60ab080f906d768 as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/346633fda9724350a60ab080f906d768 2023-10-02T12:37:09,925 INFO [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 232c4115fe6f97bf9747948e2cf60e81/session of 232c4115fe6f97bf9747948e2cf60e81 into 346633fda9724350a60ab080f906d768(size=5.6 K), total size for store is 5.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-10-02T12:37:09,925 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 232c4115fe6f97bf9747948e2cf60e81: 2023-10-02T12:37:09,925 INFO [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81., storeName=232c4115fe6f97bf9747948e2cf60e81/session, priority=13, startTime=1696250229860; duration=0sec 2023-10-02T12:37:09,926 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-10-02T12:37:09,926 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 232c4115fe6f97bf9747948e2cf60e81:session 2023-10-02T12:37:10,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=59 2023-10-02T12:37:10,116 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-10-02T12:37:10,117 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:37:10,122 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:37:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:37:10,125 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:37:10,125 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: REQUEST 2023-10-02T12:37:10,127 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:37:10,132 ERROR [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(246): Backup backup_1696250224034 failed. Backup session finished. Status: FAILURE 2023-10-02T12:37:10,133 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:10,133 ERROR [Listener at jenkins-hbase3.apache.org/34907 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.runBackupAndFailAtStage(TestFullBackupWithFailures.java:67) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.testFullBackupWithFailures(TestFullBackupWithFailures.java:55) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_2 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:211) ~[test-classes/:?] ... 32 more 2023-10-02T12:37:10,190 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] backup.TestFullBackupWithFailures(54): Running stage 3 2023-10-02T12:37:10,216 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49463f04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:37:10,217 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:37:10,219 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52654, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:37:10,223 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:37:10,225 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60928, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:37:10,267 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:37:10,271 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:37:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250230271 (current time:1696250230271). 2023-10-02T12:37:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:37:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:37:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b6cf41b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:37:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:37:10,288 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60936, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:37:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:37:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=62, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:37:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-10-02T12:37:10,292 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:37:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-10-02T12:37:10,293 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:37:10,295 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot, current working dir path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:37:10,297 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:37:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-10-02T12:37:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-10-02T12:37:10,704 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:37:10,704 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure 232c4115fe6f97bf9747948e2cf60e81}] 2023-10-02T12:37:10,706 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:10,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37985 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=63 2023-10-02T12:37:10,858 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:10,858 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2819): Flushing 232c4115fe6f97bf9747948e2cf60e81 2/2 column families, dataSize=341 B heapSize=1.05 KB 2023-10-02T12:37:10,864 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/42d5514f478c436a97d425898f6f3106 is 256, key is session:backup_1696250224034/session:context/1696250230126/Put/seqid=0 2023-10-02T12:37:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-10-02T12:37:11,269 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=341 B at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/42d5514f478c436a97d425898f6f3106 2023-10-02T12:37:11,277 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/42d5514f478c436a97d425898f6f3106 as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/42d5514f478c436a97d425898f6f3106 2023-10-02T12:37:11,282 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/42d5514f478c436a97d425898f6f3106, entries=2, sequenceid=48, filesize=5.2 K 2023-10-02T12:37:11,283 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(3022): Finished flush of dataSize ~341 B/341, heapSize ~808 B/808, currentSize=0 B/0 for 232c4115fe6f97bf9747948e2cf60e81 in 425ms, sequenceid=48, compaction requested=false 2023-10-02T12:37:11,283 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2520): Flush status journal for 232c4115fe6f97bf9747948e2cf60e81: 2023-10-02T12:37:11,283 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. for snapshot_backup_system completed. 2023-10-02T12:37:11,283 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:37:11,284 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:37:11,284 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:37:11,284 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/346633fda9724350a60ab080f906d768, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/42d5514f478c436a97d425898f6f3106] hfiles 2023-10-02T12:37:11,284 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/346633fda9724350a60ab080f906d768 for snapshot=snapshot_backup_system 2023-10-02T12:37:11,284 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/42d5514f478c436a97d425898f6f3106 for snapshot=snapshot_backup_system 2023-10-02T12:37:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-10-02T12:37:11,692 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:11,692 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=63 2023-10-02T12:37:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster(3995): Remote procedure done, pid=63 2023-10-02T12:37:11,693 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:11,694 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:11,696 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=63, resume processing ppid=62 2023-10-02T12:37:11,696 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:37:11,696 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=63, ppid=62, state=SUCCESS; SnapshotRegionProcedure 232c4115fe6f97bf9747948e2cf60e81 in 991 msec 2023-10-02T12:37:11,697 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:37:11,698 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:37:11,698 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:37:11,698 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:37:12,114 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:37:12,122 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:37:12,123 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:37:12,124 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:37:12,124 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-10-02T12:37:12,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=62, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8340 sec 2023-10-02T12:37:12,211 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:37:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-10-02T12:37:12,400 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(115): Backup backup_1696250230252 started at 1696250232400. 2023-10-02T12:37:12,402 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(119): Backup session backup_1696250230252 has been started. 2023-10-02T12:37:12,405 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] backup.TestBackupBase$FullTableBackupClientForTest(205): Execute roll log procedure for full backup ... 2023-10-02T12:37:12,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:37:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-10-02T12:37:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:37:12,406 INFO [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:37:12,406 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:37:12,407 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:37:12,407 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:37:12,408 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:37:12,410 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:12,410 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:37:12,410 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:37:12,411 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:37:12,411 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:37:12,411 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:37:12,411 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:37:12,412 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:37:12,412 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:37:12,412 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-10-02T12:37:12,412 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:37:12,412 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:37:12,412 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:37:12,413 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:37:12,413 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:37:12,413 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,37985,1696250191566' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:37:12,416 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:12,416 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:37:12,416 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:37:12,416 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:37:12,416 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:37:12,417 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:37:12,417 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:37:12,417 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:37:12,418 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:12,419 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:37:12,420 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,37985,1696250191566' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:37:12,420 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@58852771[Count = 0] remaining members to acquire global barrier 2023-10-02T12:37:12,420 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:37:12,420 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:37:12,421 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:37:12,421 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:37:12,421 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:37:12,422 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:37:12,422 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:37:12,422 DEBUG [rs(jenkins-hbase3.apache.org,37985,1696250191566)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:12,422 INFO [rs(jenkins-hbase3.apache.org,37985,1696250191566)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696250226246 highest: 1696250226246 on jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:12,422 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37985%2C1696250191566.meta:.meta(num 1696250226222) roll requested 2023-10-02T12:37:12,438 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.meta.1696250232422.meta, exclude list is [], retry=0 2023-10-02T12:37:12,444 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33097,DS-0c137cf7-1875-4349-b1e4-5237971c808a,DISK] 2023-10-02T12:37:12,446 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.meta.1696250226222.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.meta.1696250232422.meta 2023-10-02T12:37:12,446 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33097,DS-0c137cf7-1875-4349-b1e4-5237971c808a,DISK]] 2023-10-02T12:37:12,446 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.meta.1696250226222.meta is not closed yet, will try archiving it next time 2023-10-02T12:37:12,446 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37985%2C1696250191566:(num 1696250226246) roll requested 2023-10-02T12:37:12,460 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.1696250232446, exclude list is [], retry=0 2023-10-02T12:37:12,462 DEBUG [rs(jenkins-hbase3.apache.org,37985,1696250191566)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-10-02T12:37:12,462 INFO [rs(jenkins-hbase3.apache.org,37985,1696250191566)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696250232446 on jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:12,465 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33097,DS-0c137cf7-1875-4349-b1e4-5237971c808a,DISK] 2023-10-02T12:37:12,467 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.1696250226246 with entries=13, filesize=3.39 KB; new WAL /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.1696250232446 2023-10-02T12:37:12,467 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33097,DS-0c137cf7-1875-4349-b1e4-5237971c808a,DISK]] 2023-10-02T12:37:12,468 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.1696250226246 is not closed yet, will try archiving it next time 2023-10-02T12:37:12,475 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:37:12,475 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,37985,1696250191566' in zk 2023-10-02T12:37:12,478 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:37:12,478 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:37:12,478 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:12,478 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:37:12,478 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:37:12,479 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:37:12,480 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:37:12,480 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:37:12,481 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:37:12,481 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:12,481 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:37:12,482 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:37:12,482 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:12,482 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,37985,1696250191566': 2023-10-02T12:37:12,483 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,37985,1696250191566' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:37:12,483 INFO [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:37:12,483 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:37:12,483 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:37:12,483 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:37:12,483 INFO [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:37:12,486 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:37:12,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:37:12,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:37:12,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:37:12,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:37:12,486 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:37:12,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:37:12,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:37:12,487 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:37:12,488 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:37:12,489 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:37:12,489 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:37:12,491 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:12,493 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:37:12,493 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:37:12,493 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:37:12,493 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:37:12,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:37:12,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:37:12,494 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:37:12,494 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:37:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:37:12,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696250232597_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } 2023-10-02T12:37:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250232597 (current time:1696250232597). 2023-10-02T12:37:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:37:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696250232597_default_test-1696250203711 VERSION not specified, setting to 2 2023-10-02T12:37:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c6f9792, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:37:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:37:12,607 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60940, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:37:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:37:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696250232597_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } 2023-10-02T12:37:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696250232597_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-10-02T12:37:12,611 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696250232597_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:37:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-10-02T12:37:12,612 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696250232597_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:37:12,615 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot, current working dir path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_1696250232597_default_test-1696250203711 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:37:12,617 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696250232597_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:37:12,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-10-02T12:37:12,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-10-02T12:37:13,024 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696250232597_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:37:13,025 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 0d8a4463d318b78a2ee236eba02d8a78}] 2023-10-02T12:37:13,026 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 0d8a4463d318b78a2ee236eba02d8a78 2023-10-02T12:37:13,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37985 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=65 2023-10-02T12:37:13,178 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. 2023-10-02T12:37:13,179 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2819): Flushing 0d8a4463d318b78a2ee236eba02d8a78 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-10-02T12:37:13,203 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/default/test-1696250203711/0d8a4463d318b78a2ee236eba02d8a78/.tmp/f/7287d7a48b5c4307947a2ab13e8623a8 is 37, key is row10/f:q1/1696250205478/Put/seqid=0 2023-10-02T12:37:13,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-10-02T12:37:13,608 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/default/test-1696250203711/0d8a4463d318b78a2ee236eba02d8a78/.tmp/f/7287d7a48b5c4307947a2ab13e8623a8 2023-10-02T12:37:13,615 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/default/test-1696250203711/0d8a4463d318b78a2ee236eba02d8a78/.tmp/f/7287d7a48b5c4307947a2ab13e8623a8 as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/default/test-1696250203711/0d8a4463d318b78a2ee236eba02d8a78/f/7287d7a48b5c4307947a2ab13e8623a8 2023-10-02T12:37:13,621 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/default/test-1696250203711/0d8a4463d318b78a2ee236eba02d8a78/f/7287d7a48b5c4307947a2ab13e8623a8, entries=99, sequenceid=103, filesize=8.2 K 2023-10-02T12:37:13,622 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 0d8a4463d318b78a2ee236eba02d8a78 in 443ms, sequenceid=103, compaction requested=false 2023-10-02T12:37:13,623 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2520): Flush status journal for 0d8a4463d318b78a2ee236eba02d8a78: 2023-10-02T12:37:13,623 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. for snapshot_1696250232597_default_test-1696250203711 completed. 2023-10-02T12:37:13,623 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(240): Storing 'test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78.' region-info for snapshot=snapshot_1696250232597_default_test-1696250203711 2023-10-02T12:37:13,623 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:37:13,623 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/default/test-1696250203711/0d8a4463d318b78a2ee236eba02d8a78/f/7287d7a48b5c4307947a2ab13e8623a8] hfiles 2023-10-02T12:37:13,623 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/default/test-1696250203711/0d8a4463d318b78a2ee236eba02d8a78/f/7287d7a48b5c4307947a2ab13e8623a8 for snapshot=snapshot_1696250232597_default_test-1696250203711 2023-10-02T12:37:13,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-10-02T12:37:14,029 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. 2023-10-02T12:37:14,029 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=65 2023-10-02T12:37:14,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster(3995): Remote procedure done, pid=65 2023-10-02T12:37:14,030 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696250232597_default_test-1696250203711 on region 0d8a4463d318b78a2ee236eba02d8a78 2023-10-02T12:37:14,031 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 0d8a4463d318b78a2ee236eba02d8a78 2023-10-02T12:37:14,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=65, resume processing ppid=64 2023-10-02T12:37:14,033 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696250232597_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:37:14,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=65, ppid=64, state=SUCCESS; SnapshotRegionProcedure 0d8a4463d318b78a2ee236eba02d8a78 in 1.0070 sec 2023-10-02T12:37:14,034 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696250232597_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:37:14,034 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696250232597_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:37:14,034 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696250232597_default_test-1696250203711 2023-10-02T12:37:14,035 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_1696250232597_default_test-1696250203711 2023-10-02T12:37:14,447 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696250232597_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:37:14,453 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696250232597_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:37:14,453 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_1696250232597_default_test-1696250203711 to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/snapshot_1696250232597_default_test-1696250203711 2023-10-02T12:37:14,454 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696250232597_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:37:14,454 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696250232597_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-10-02T12:37:14,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_1696250232597_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } in 1.8460 sec 2023-10-02T12:37:14,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-10-02T12:37:14,719 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696250234718_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } 2023-10-02T12:37:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250234719 (current time:1696250234719). 2023-10-02T12:37:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:37:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696250234718_ns2_test-16962502037111 VERSION not specified, setting to 2 2023-10-02T12:37:14,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626ee650, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:37:14,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:37:14,730 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60952, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:37:14,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:37:14,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696250234718_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } 2023-10-02T12:37:14,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696250234718_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-10-02T12:37:14,734 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696250234718_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:37:14,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-10-02T12:37:14,735 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696250234718_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:37:14,737 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot, current working dir path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_1696250234718_ns2_test-16962502037111 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:37:14,739 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696250234718_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:37:14,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-10-02T12:37:15,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-10-02T12:37:15,147 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696250234718_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:37:15,147 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure b6da6887b5f5557b8eaf374a87ad833b}] 2023-10-02T12:37:15,151 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure b6da6887b5f5557b8eaf374a87ad833b 2023-10-02T12:37:15,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37985 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=67 2023-10-02T12:37:15,303 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. 2023-10-02T12:37:15,304 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(2819): Flushing b6da6887b5f5557b8eaf374a87ad833b 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-10-02T12:37:15,319 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns2/test-16962502037111/b6da6887b5f5557b8eaf374a87ad833b/.tmp/f/a89c04d42b29401c925078c79710cedf is 37, key is row10/f:q1/1696250206241/Put/seqid=0 2023-10-02T12:37:15,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-10-02T12:37:15,724 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns2/test-16962502037111/b6da6887b5f5557b8eaf374a87ad833b/.tmp/f/a89c04d42b29401c925078c79710cedf 2023-10-02T12:37:15,731 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns2/test-16962502037111/b6da6887b5f5557b8eaf374a87ad833b/.tmp/f/a89c04d42b29401c925078c79710cedf as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns2/test-16962502037111/b6da6887b5f5557b8eaf374a87ad833b/f/a89c04d42b29401c925078c79710cedf 2023-10-02T12:37:15,736 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns2/test-16962502037111/b6da6887b5f5557b8eaf374a87ad833b/f/a89c04d42b29401c925078c79710cedf, entries=99, sequenceid=103, filesize=8.2 K 2023-10-02T12:37:15,737 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for b6da6887b5f5557b8eaf374a87ad833b in 433ms, sequenceid=103, compaction requested=false 2023-10-02T12:37:15,739 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(2520): Flush status journal for b6da6887b5f5557b8eaf374a87ad833b: 2023-10-02T12:37:15,739 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. for snapshot_1696250234718_ns2_test-16962502037111 completed. 2023-10-02T12:37:15,739 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b.' region-info for snapshot=snapshot_1696250234718_ns2_test-16962502037111 2023-10-02T12:37:15,739 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:37:15,739 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns2/test-16962502037111/b6da6887b5f5557b8eaf374a87ad833b/f/a89c04d42b29401c925078c79710cedf] hfiles 2023-10-02T12:37:15,739 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns2/test-16962502037111/b6da6887b5f5557b8eaf374a87ad833b/f/a89c04d42b29401c925078c79710cedf for snapshot=snapshot_1696250234718_ns2_test-16962502037111 2023-10-02T12:37:15,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-10-02T12:37:16,074 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-10-02T12:37:16,074 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-10-02T12:37:16,144 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. 2023-10-02T12:37:16,145 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=67 2023-10-02T12:37:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster(3995): Remote procedure done, pid=67 2023-10-02T12:37:16,145 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696250234718_ns2_test-16962502037111 on region b6da6887b5f5557b8eaf374a87ad833b 2023-10-02T12:37:16,148 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure b6da6887b5f5557b8eaf374a87ad833b 2023-10-02T12:37:16,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=67, resume processing ppid=66 2023-10-02T12:37:16,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=67, ppid=66, state=SUCCESS; SnapshotRegionProcedure b6da6887b5f5557b8eaf374a87ad833b in 1.0020 sec 2023-10-02T12:37:16,150 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696250234718_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:37:16,151 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696250234718_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:37:16,152 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696250234718_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:37:16,152 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696250234718_ns2_test-16962502037111 2023-10-02T12:37:16,153 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_1696250234718_ns2_test-16962502037111 2023-10-02T12:37:16,578 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696250234718_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:37:16,584 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696250234718_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:37:16,585 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_1696250234718_ns2_test-16962502037111 to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/snapshot_1696250234718_ns2_test-16962502037111 2023-10-02T12:37:16,586 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696250234718_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:37:16,586 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696250234718_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-10-02T12:37:16,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=66, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696250234718_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } in 1.8540 sec 2023-10-02T12:37:16,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-10-02T12:37:16,840 ERROR [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(228): Unexpected BackupException : Failed stage stage_3 in testing java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:223) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.runBackupAndFailAtStage(TestFullBackupWithFailures.java:67) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.testFullBackupWithFailures(TestFullBackupWithFailures.java:55) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:37:16,841 ERROR [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(240): BackupId=backup_1696250230252,startts=1696250232400,failedts=1696250236841,failedphase=SNAPSHOT,failedmessage=Failed stage stage_3 in testing 2023-10-02T12:37:16,841 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:37:16,841 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696250232597_default_test-1696250203711 2023-10-02T12:37:16,842 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696250232597_default_test-1696250203711" 2023-10-02T12:37:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696250232597_default_test-1696250203711 2023-10-02T12:37:16,845 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696250232597_default_test-1696250203711 for backup backup_1696250230252 succeeded. 2023-10-02T12:37:16,845 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696250234718_ns2_test-16962502037111 2023-10-02T12:37:16,846 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696250234718_ns2_test-16962502037111" 2023-10-02T12:37:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696250234718_ns2_test-16962502037111 2023-10-02T12:37:16,849 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696250234718_ns2_test-16962502037111 for backup backup_1696250230252 succeeded. 2023-10-02T12:37:16,850 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-10-02T12:37:16,855 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable backup:system 2023-10-02T12:37:16,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=68, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-10-02T12:37:16,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-10-02T12:37:16,860 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250236859"}]},"ts":"1696250236859"} 2023-10-02T12:37:16,861 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-10-02T12:37:16,863 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-10-02T12:37:16,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, UNASSIGN}] 2023-10-02T12:37:16,865 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, UNASSIGN 2023-10-02T12:37:16,865 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=69 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:16,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE; CloseRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566}] 2023-10-02T12:37:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-10-02T12:37:17,018 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] handler.UnassignRegionHandler(119): Close 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:17,019 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1668): Closing 232c4115fe6f97bf9747948e2cf60e81, disabling compactions & flushes 2023-10-02T12:37:17,019 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1690): Closing region backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:17,019 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:17,019 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. after waiting 0 ms 2023-10-02T12:37:17,019 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:17,019 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(2819): Flushing 232c4115fe6f97bf9747948e2cf60e81 2/2 column families, dataSize=401 B heapSize=1.11 KB 2023-10-02T12:37:17,034 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/meta/4e74def075f645098e610d2401c1cc8b is 116, key is rslogts:hdfs://localhost:40817/backupUT\x00jenkins-hbase3.apache.org:37985/meta:rs-log-ts/1696250232474/Put/seqid=0 2023-10-02T12:37:17,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-10-02T12:37:17,439 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=187 B at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/meta/4e74def075f645098e610d2401c1cc8b 2023-10-02T12:37:17,450 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/1bc78cd3467a428fac3a10aa939b8504 is 218, key is session:backup_1696250230252/session:context/1696250232401/Put/seqid=0 2023-10-02T12:37:17,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-10-02T12:37:17,855 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/1bc78cd3467a428fac3a10aa939b8504 2023-10-02T12:37:17,863 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/meta/4e74def075f645098e610d2401c1cc8b as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/4e74def075f645098e610d2401c1cc8b 2023-10-02T12:37:17,872 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/4e74def075f645098e610d2401c1cc8b, entries=2, sequenceid=54, filesize=5.3 K 2023-10-02T12:37:17,873 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/1bc78cd3467a428fac3a10aa939b8504 as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/1bc78cd3467a428fac3a10aa939b8504 2023-10-02T12:37:17,880 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/1bc78cd3467a428fac3a10aa939b8504, entries=1, sequenceid=54, filesize=5.2 K 2023-10-02T12:37:17,882 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(3022): Finished flush of dataSize ~401 B/401, heapSize ~1.08 KB/1104, currentSize=0 B/0 for 232c4115fe6f97bf9747948e2cf60e81 in 862ms, sequenceid=54, compaction requested=true 2023-10-02T12:37:17,891 DEBUG [StoreCloser-backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/670bddd78f654d84b6033b3d735ecaee, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/dcd80847668348169ebbbaf8dcaff038] to archive 2023-10-02T12:37:17,893 DEBUG [StoreCloser-backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-10-02T12:37:17,897 DEBUG [StoreCloser-backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646 to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/archive/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/547c8c6158404697bd5ec227f6f3d646 2023-10-02T12:37:17,899 DEBUG [StoreCloser-backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/670bddd78f654d84b6033b3d735ecaee to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/archive/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/670bddd78f654d84b6033b3d735ecaee 2023-10-02T12:37:17,902 DEBUG [StoreCloser-backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/dcd80847668348169ebbbaf8dcaff038 to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/archive/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/dcd80847668348169ebbbaf8dcaff038 2023-10-02T12:37:17,956 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/recovered.edits/57.seqid, newMaxSeqId=57, maxSeqId=41 2023-10-02T12:37:17,956 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:37:17,958 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1909): Closed backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:17,958 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] regionserver.HRegion(1622): Region close journal for 232c4115fe6f97bf9747948e2cf60e81: 2023-10-02T12:37:17,960 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=70}] handler.UnassignRegionHandler(163): Closed 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:17,962 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=69 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=CLOSED 2023-10-02T12:37:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-10-02T12:37:17,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=70, resume processing ppid=69 2023-10-02T12:37:17,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=70, ppid=69, state=SUCCESS; CloseRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566 in 1.0990 sec 2023-10-02T12:37:17,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=69, resume processing ppid=68 2023-10-02T12:37:17,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=69, ppid=68, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, UNASSIGN in 1.1050 sec 2023-10-02T12:37:17,973 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250237972"}]},"ts":"1696250237972"} 2023-10-02T12:37:17,978 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-10-02T12:37:17,980 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-10-02T12:37:17,994 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=68, state=SUCCESS; DisableTableProcedure table=backup:system in 1.1310 sec 2023-10-02T12:37:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-10-02T12:37:18,976 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1696250238975 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:37:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250238976 (current time:1696250238976). 2023-10-02T12:37:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:37:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1696250238975 VERSION not specified, setting to 2 2023-10-02T12:37:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dbea76e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:37:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:37:18,987 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60966, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:37:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:37:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=71, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250238975 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:37:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250238975 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 71 2023-10-02T12:37:18,991 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250238975 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:37:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-10-02T12:37:18,992 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250238975 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:37:18,995 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot, current working dir path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696250238975 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:37:18,997 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250238975 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:37:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-10-02T12:37:19,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-10-02T12:37:19,404 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250238975 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-10-02T12:37:19,405 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-10-02T12:37:19,405 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-10-02T12:37:19,405 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:19,406 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/4e74def075f645098e610d2401c1cc8b] hfiles 2023-10-02T12:37:19,406 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/4e74def075f645098e610d2401c1cc8b 2023-10-02T12:37:19,407 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:19,408 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/1bc78cd3467a428fac3a10aa939b8504, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/346633fda9724350a60ab080f906d768, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/42d5514f478c436a97d425898f6f3106] hfiles 2023-10-02T12:37:19,408 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/1bc78cd3467a428fac3a10aa939b8504 2023-10-02T12:37:19,409 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/346633fda9724350a60ab080f906d768 2023-10-02T12:37:19,409 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/42d5514f478c436a97d425898f6f3106 2023-10-02T12:37:19,415 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-10-02T12:37:19,417 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250238975 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:37:19,418 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250238975 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:37:19,418 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1696250238975 2023-10-02T12:37:19,419 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696250238975 2023-10-02T12:37:19,430 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250238975 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:37:19,439 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250238975 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:37:19,439 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696250238975 to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1696250238975 2023-10-02T12:37:19,440 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=71, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250238975 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:37:19,441 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250238975 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 71 2023-10-02T12:37:19,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=71, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=71, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250238975 table=backup:system type=FLUSH ttl=0 } in 452 msec 2023-10-02T12:37:19,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=71 2023-10-02T12:37:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=72, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696250230271 type: FLUSH version: 2 ttl: 0 ) 2023-10-02T12:37:19,605 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-10-02T12:37:19,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-10-02T12:37:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-10-02T12:37:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-10-02T12:37:20,018 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/.tabledesc/.tableinfo.0000000004.635 2023-10-02T12:37:20,018 INFO [PEWorker-2 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/.tabledesc/.tableinfo.0000000005.635 2023-10-02T12:37:20,019 INFO [PEWorker-2 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:37:20,021 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696250230271 type: FLUSH version: 2 ttl: 0 2023-10-02T12:37:20,022 DEBUG [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system 2023-10-02T12:37:20,024 DEBUG [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-10-02T12:37:20,024 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:20,033 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/4e74def075f645098e610d2401c1cc8b to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/archive/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/4e74def075f645098e610d2401c1cc8b 2023-10-02T12:37:20,033 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:20,036 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/1bc78cd3467a428fac3a10aa939b8504 to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/archive/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/1bc78cd3467a428fac3a10aa939b8504 2023-10-02T12:37:20,036 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:20,036 INFO [PEWorker-2 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696250230271 type: FLUSH version: 2 ttl: 0 2023-10-02T12:37:20,037 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696250240037"}]},"ts":"9223372036854775807"} 2023-10-02T12:37:20,040 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:37:20,040 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:37:20,040 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1696250240038"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250240038"}]},"ts":"1696250240038"} 2023-10-02T12:37:20,041 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:37:20,041 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-10-02T12:37:20,041 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:37:20,041 INFO [PEWorker-2 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-10-02T12:37:20,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=72, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696250230271 type: FLUSH version: 2 ttl: 0 ) in 438 msec 2023-10-02T12:37:20,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-10-02T12:37:20,211 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1696250238975" 2023-10-02T12:37:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1696250238975 2023-10-02T12:37:20,215 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster$11(2631): Client=jenkins//172.31.12.81 enable backup:system 2023-10-02T12:37:20,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=73, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-10-02T12:37:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=73 2023-10-02T12:37:20,219 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-10-02T12:37:20,219 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250240219"}]},"ts":"1696250240219"} 2023-10-02T12:37:20,220 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-10-02T12:37:20,222 DEBUG [PEWorker-1 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-10-02T12:37:20,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN}] 2023-10-02T12:37:20,224 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN 2023-10-02T12:37:20,225 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-10-02T12:37:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=73 2023-10-02T12:37:20,376 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=74 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:20,378 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE; OpenRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566}] 2023-10-02T12:37:20,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=73 2023-10-02T12:37:20,533 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(130): Open backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:20,533 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7254): Opening region: {ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:37:20,534 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:37:20,534 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:20,534 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(888): Instantiated backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:37:20,534 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7296): checking encryption for 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:20,534 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7299): checking classloading for 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:20,535 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:20,536 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 232c4115fe6f97bf9747948e2cf60e81 columnFamilyName meta 2023-10-02T12:37:20,536 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:20,537 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(324): Store=232c4115fe6f97bf9747948e2cf60e81/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:37:20,537 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:20,538 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 232c4115fe6f97bf9747948e2cf60e81 columnFamilyName session 2023-10-02T12:37:20,538 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:20,544 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/346633fda9724350a60ab080f906d768 2023-10-02T12:37:20,549 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/42d5514f478c436a97d425898f6f3106 2023-10-02T12:37:20,549 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(324): Store=232c4115fe6f97bf9747948e2cf60e81/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:37:20,550 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:20,551 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:20,553 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:37:20,554 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1076): writing seq id for 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:20,555 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): Opened 232c4115fe6f97bf9747948e2cf60e81; next sequenceid=58; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11099569440, jitterRate=0.03372795879840851}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:37:20,555 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(992): Region open journal for 232c4115fe6f97bf9747948e2cf60e81: 2023-10-02T12:37:20,556 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81., pid=75, masterSystemTime=1696250240529 2023-10-02T12:37:20,558 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:20,558 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(158): Opened backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:20,558 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=74 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=OPEN, openSeqNum=58, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:20,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=75, resume processing ppid=74 2023-10-02T12:37:20,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=75, ppid=74, state=SUCCESS; OpenRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566 in 182 msec 2023-10-02T12:37:20,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=74, resume processing ppid=73 2023-10-02T12:37:20,563 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=74, ppid=73, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN in 339 msec 2023-10-02T12:37:20,564 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250240563"}]},"ts":"1696250240563"} 2023-10-02T12:37:20,565 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-10-02T12:37:20,567 INFO [PEWorker-5 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-10-02T12:37:20,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=73, state=SUCCESS; EnableTableProcedure table=backup:system in 353 msec 2023-10-02T12:37:20,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=73 2023-10-02T12:37:20,821 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-10-02T12:37:20,821 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:37:20,826 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:37:20,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:37:20,829 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:37:20,829 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: SNAPSHOT 2023-10-02T12:37:20,831 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:37:20,837 ERROR [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(246): Backup backup_1696250230252 failed. Backup session finished. Status: FAILURE 2023-10-02T12:37:20,837 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:20,837 ERROR [Listener at jenkins-hbase3.apache.org/34907 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.runBackupAndFailAtStage(TestFullBackupWithFailures.java:67) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.testFullBackupWithFailures(TestFullBackupWithFailures.java:55) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_3 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:223) ~[test-classes/:?] ... 32 more 2023-10-02T12:37:20,881 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] backup.TestFullBackupWithFailures(54): Running stage 4 2023-10-02T12:37:20,902 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c7da88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:37:20,903 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:37:20,904 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51062, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:37:20,908 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:37:20,910 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37622, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:37:20,949 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:37:20,953 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:37:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250240953 (current time:1696250240953). 2023-10-02T12:37:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:37:20,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:37:20,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ccc1e0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:37:20,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:37:20,964 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37634, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:37:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:20,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:37:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=76, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:37:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 76 2023-10-02T12:37:20,968 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=76, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:37:20,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=76 2023-10-02T12:37:20,969 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=76, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:37:20,972 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot, current working dir path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:37:20,974 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=76, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:37:21,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=76 2023-10-02T12:37:21,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=76 2023-10-02T12:37:21,380 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=76, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:37:21,380 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; SnapshotRegionProcedure 232c4115fe6f97bf9747948e2cf60e81}] 2023-10-02T12:37:21,382 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=77, ppid=76, state=RUNNABLE; SnapshotRegionProcedure 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:21,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37985 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=77 2023-10-02T12:37:21,533 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:21,534 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] regionserver.HRegion(2819): Flushing 232c4115fe6f97bf9747948e2cf60e81 2/2 column families, dataSize=440 B heapSize=1.15 KB 2023-10-02T12:37:21,548 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/5dc67774b6954153bced22444bed4827 is 355, key is session:backup_1696250230252/session:context/1696250240830/Put/seqid=0 2023-10-02T12:37:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=76 2023-10-02T12:37:21,953 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=440 B at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/5dc67774b6954153bced22444bed4827 2023-10-02T12:37:21,959 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/5dc67774b6954153bced22444bed4827 as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/5dc67774b6954153bced22444bed4827 2023-10-02T12:37:21,964 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/5dc67774b6954153bced22444bed4827, entries=2, sequenceid=63, filesize=5.3 K 2023-10-02T12:37:21,965 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] regionserver.HRegion(3022): Finished flush of dataSize ~440 B/440, heapSize ~904 B/904, currentSize=0 B/0 for 232c4115fe6f97bf9747948e2cf60e81 in 431ms, sequenceid=63, compaction requested=true 2023-10-02T12:37:21,965 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] regionserver.HRegion(2520): Flush status journal for 232c4115fe6f97bf9747948e2cf60e81: 2023-10-02T12:37:21,965 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. for snapshot_backup_system completed. 2023-10-02T12:37:21,965 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:37:21,965 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:37:21,965 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:37:21,965 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/346633fda9724350a60ab080f906d768, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/42d5514f478c436a97d425898f6f3106, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/5dc67774b6954153bced22444bed4827] hfiles 2023-10-02T12:37:21,966 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/346633fda9724350a60ab080f906d768 for snapshot=snapshot_backup_system 2023-10-02T12:37:21,966 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/42d5514f478c436a97d425898f6f3106 for snapshot=snapshot_backup_system 2023-10-02T12:37:21,966 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/5dc67774b6954153bced22444bed4827 for snapshot=snapshot_backup_system 2023-10-02T12:37:21,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:21,971 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=77 2023-10-02T12:37:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster(3995): Remote procedure done, pid=77 2023-10-02T12:37:21,971 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:21,972 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=77, ppid=76, state=RUNNABLE; SnapshotRegionProcedure 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:21,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=77, resume processing ppid=76 2023-10-02T12:37:21,974 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=76, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:37:21,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=77, ppid=76, state=SUCCESS; SnapshotRegionProcedure 232c4115fe6f97bf9747948e2cf60e81 in 593 msec 2023-10-02T12:37:21,975 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=76, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:37:21,976 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=76, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:37:21,976 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:37:21,976 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:37:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=76 2023-10-02T12:37:22,385 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=76, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:37:22,392 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=76, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:37:22,392 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:37:22,393 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=76, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:37:22,393 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 76 2023-10-02T12:37:22,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=76, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=76, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.4270 sec 2023-10-02T12:37:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=76 2023-10-02T12:37:23,075 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(115): Backup backup_1696250240933 started at 1696250243075. 2023-10-02T12:37:23,077 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(119): Backup session backup_1696250240933 has been started. 2023-10-02T12:37:23,079 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] backup.TestBackupBase$FullTableBackupClientForTest(205): Execute roll log procedure for full backup ... 2023-10-02T12:37:23,079 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:37:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-10-02T12:37:23,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:37:23,080 INFO [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:37:23,080 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:37:23,081 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:37:23,081 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:37:23,081 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:37:23,083 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:23,083 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:37:23,083 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:37:23,083 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:37:23,083 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:37:23,084 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:37:23,084 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:37:23,084 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:37:23,084 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:37:23,084 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-10-02T12:37:23,084 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:37:23,084 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:37:23,084 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:37:23,085 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:37:23,085 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:37:23,085 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,37985,1696250191566' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:37:23,088 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:37:23,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:23,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:37:23,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:37:23,088 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:37:23,089 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:37:23,089 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:37:23,089 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:37:23,089 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:23,090 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:37:23,090 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,37985,1696250191566' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:37:23,090 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@21ff5b33[Count = 0] remaining members to acquire global barrier 2023-10-02T12:37:23,090 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:37:23,090 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:37:23,091 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:37:23,091 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:37:23,091 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:37:23,092 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:37:23,092 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:37:23,092 DEBUG [rs(jenkins-hbase3.apache.org,37985,1696250191566)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:23,092 INFO [rs(jenkins-hbase3.apache.org,37985,1696250191566)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696250232446 highest: 1696250232446 on jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:23,092 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37985%2C1696250191566.meta:.meta(num 1696250232422) roll requested 2023-10-02T12:37:23,103 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.meta.1696250243092.meta, exclude list is [], retry=0 2023-10-02T12:37:23,107 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33097,DS-0c137cf7-1875-4349-b1e4-5237971c808a,DISK] 2023-10-02T12:37:23,110 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.meta.1696250232422.meta with entries=10, filesize=2.50 KB; new WAL /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.meta.1696250243092.meta 2023-10-02T12:37:23,110 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33097,DS-0c137cf7-1875-4349-b1e4-5237971c808a,DISK]] 2023-10-02T12:37:23,110 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.meta.1696250232422.meta is not closed yet, will try archiving it next time 2023-10-02T12:37:23,110 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C37985%2C1696250191566:(num 1696250232446) roll requested 2023-10-02T12:37:23,122 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.1696250243110, exclude list is [], retry=0 2023-10-02T12:37:23,126 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33097,DS-0c137cf7-1875-4349-b1e4-5237971c808a,DISK] 2023-10-02T12:37:23,127 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.1696250232446 with entries=16, filesize=4.15 KB; new WAL /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.1696250243110 2023-10-02T12:37:23,127 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:33097,DS-0c137cf7-1875-4349-b1e4-5237971c808a,DISK]] 2023-10-02T12:37:23,127 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.1696250232446 is not closed yet, will try archiving it next time 2023-10-02T12:37:23,129 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.1696250226246 to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/oldWALs/jenkins-hbase3.apache.org%2C37985%2C1696250191566.1696250226246 2023-10-02T12:37:23,129 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.1696250232446 not finished, retry = 0 2023-10-02T12:37:23,132 DEBUG [rs(jenkins-hbase3.apache.org,37985,1696250191566)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-10-02T12:37:23,132 INFO [rs(jenkins-hbase3.apache.org,37985,1696250191566)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696250243110 on jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:23,140 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:37:23,140 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,37985,1696250191566' in zk 2023-10-02T12:37:23,143 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:37:23,143 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:37:23,143 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:23,143 DEBUG [member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:37:23,143 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:37:23,143 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:37:23,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:37:23,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:37:23,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:37:23,145 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:23,145 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:37:23,145 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:37:23,146 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:23,146 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,37985,1696250191566': 2023-10-02T12:37:23,146 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,37985,1696250191566' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:37:23,146 INFO [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:37:23,146 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:37:23,146 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:37:23,146 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:37:23,146 INFO [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:37:23,148 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:37:23,148 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:37:23,148 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:37:23,148 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:37:23,148 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:37:23,148 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:37:23,148 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:37:23,148 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:37:23,149 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:37:23,149 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:37:23,149 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:37:23,149 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:37:23,150 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:23,150 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:37:23,150 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:37:23,151 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:23,155 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:37:23,155 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:37:23,155 DEBUG [(jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:37:23,155 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:37:23,155 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:37:23,156 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:37:23,156 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:37:23,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:37:23,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696250243258_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } 2023-10-02T12:37:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250243259 (current time:1696250243259). 2023-10-02T12:37:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:37:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696250243258_default_test-1696250203711 VERSION not specified, setting to 2 2023-10-02T12:37:23,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@233d3078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:37:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:37:23,268 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37642, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:37:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:23,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:37:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=78, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1696250243258_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } 2023-10-02T12:37:23,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696250243258_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 }, snapshot procedure id = 78 2023-10-02T12:37:23,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-10-02T12:37:23,272 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=78, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1696250243258_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:37:23,273 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=78, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1696250243258_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:37:23,276 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot, current working dir path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_1696250243258_default_test-1696250203711 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:37:23,278 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=78, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1696250243258_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:37:23,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-10-02T12:37:23,506 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:37:23,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-10-02T12:37:23,685 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=78, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1696250243258_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:37:23,686 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE; SnapshotRegionProcedure 0d8a4463d318b78a2ee236eba02d8a78}] 2023-10-02T12:37:23,688 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=79, ppid=78, state=RUNNABLE; SnapshotRegionProcedure 0d8a4463d318b78a2ee236eba02d8a78 2023-10-02T12:37:23,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37985 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=79 2023-10-02T12:37:23,839 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=79}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. 2023-10-02T12:37:23,840 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=79}] regionserver.HRegion(2520): Flush status journal for 0d8a4463d318b78a2ee236eba02d8a78: 2023-10-02T12:37:23,840 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=79}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. for snapshot_1696250243258_default_test-1696250203711 completed. 2023-10-02T12:37:23,840 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=79}] snapshot.SnapshotManifest(240): Storing 'test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78.' region-info for snapshot=snapshot_1696250243258_default_test-1696250203711 2023-10-02T12:37:23,840 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=79}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:37:23,840 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=79}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/default/test-1696250203711/0d8a4463d318b78a2ee236eba02d8a78/f/7287d7a48b5c4307947a2ab13e8623a8] hfiles 2023-10-02T12:37:23,840 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=79}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/default/test-1696250203711/0d8a4463d318b78a2ee236eba02d8a78/f/7287d7a48b5c4307947a2ab13e8623a8 for snapshot=snapshot_1696250243258_default_test-1696250203711 2023-10-02T12:37:23,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-10-02T12:37:24,246 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=79}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. 2023-10-02T12:37:24,246 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=79 2023-10-02T12:37:24,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster(3995): Remote procedure done, pid=79 2023-10-02T12:37:24,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696250243258_default_test-1696250203711 on region 0d8a4463d318b78a2ee236eba02d8a78 2023-10-02T12:37:24,248 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=79, ppid=78, state=RUNNABLE; SnapshotRegionProcedure 0d8a4463d318b78a2ee236eba02d8a78 2023-10-02T12:37:24,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=79, resume processing ppid=78 2023-10-02T12:37:24,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=79, ppid=78, state=SUCCESS; SnapshotRegionProcedure 0d8a4463d318b78a2ee236eba02d8a78 in 563 msec 2023-10-02T12:37:24,250 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=78, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1696250243258_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:37:24,251 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=78, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1696250243258_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:37:24,251 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=78, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1696250243258_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:37:24,251 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696250243258_default_test-1696250203711 2023-10-02T12:37:24,252 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_1696250243258_default_test-1696250203711 2023-10-02T12:37:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-10-02T12:37:24,662 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=78, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1696250243258_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:37:24,667 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=78, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1696250243258_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:37:24,668 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_1696250243258_default_test-1696250203711 to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/snapshot_1696250243258_default_test-1696250203711 2023-10-02T12:37:24,669 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=78, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1696250243258_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:37:24,670 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696250243258_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 }, snapshot procedure id = 78 2023-10-02T12:37:24,670 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=78, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=78, snapshot={ ss=snapshot_1696250243258_default_test-1696250203711 table=test-1696250203711 type=FLUSH ttl=0 } in 1.4010 sec 2023-10-02T12:37:25,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-10-02T12:37:25,379 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696250245378_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } 2023-10-02T12:37:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250245379 (current time:1696250245379). 2023-10-02T12:37:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:37:25,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696250245378_ns2_test-16962502037111 VERSION not specified, setting to 2 2023-10-02T12:37:25,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@178dbdee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:37:25,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:37:25,391 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37646, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:37:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:37:25,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=80, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1696250245378_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } 2023-10-02T12:37:25,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696250245378_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 }, snapshot procedure id = 80 2023-10-02T12:37:25,395 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=80, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1696250245378_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:37:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=80 2023-10-02T12:37:25,396 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=80, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1696250245378_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:37:25,398 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot, current working dir path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_1696250245378_ns2_test-16962502037111 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:37:25,400 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=80, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1696250245378_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:37:25,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=80 2023-10-02T12:37:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=80 2023-10-02T12:37:25,806 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=80, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1696250245378_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:37:25,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE; SnapshotRegionProcedure b6da6887b5f5557b8eaf374a87ad833b}] 2023-10-02T12:37:25,808 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=81, ppid=80, state=RUNNABLE; SnapshotRegionProcedure b6da6887b5f5557b8eaf374a87ad833b 2023-10-02T12:37:25,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37985 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=81 2023-10-02T12:37:25,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=81}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. 2023-10-02T12:37:25,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=81}] regionserver.HRegion(2520): Flush status journal for b6da6887b5f5557b8eaf374a87ad833b: 2023-10-02T12:37:25,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=81}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. for snapshot_1696250245378_ns2_test-16962502037111 completed. 2023-10-02T12:37:25,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=81}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b.' region-info for snapshot=snapshot_1696250245378_ns2_test-16962502037111 2023-10-02T12:37:25,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=81}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:37:25,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=81}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns2/test-16962502037111/b6da6887b5f5557b8eaf374a87ad833b/f/a89c04d42b29401c925078c79710cedf] hfiles 2023-10-02T12:37:25,960 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=81}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns2/test-16962502037111/b6da6887b5f5557b8eaf374a87ad833b/f/a89c04d42b29401c925078c79710cedf for snapshot=snapshot_1696250245378_ns2_test-16962502037111 2023-10-02T12:37:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=80 2023-10-02T12:37:26,367 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=81}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. 2023-10-02T12:37:26,367 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=81 2023-10-02T12:37:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster(3995): Remote procedure done, pid=81 2023-10-02T12:37:26,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696250245378_ns2_test-16962502037111 on region b6da6887b5f5557b8eaf374a87ad833b 2023-10-02T12:37:26,369 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=81, ppid=80, state=RUNNABLE; SnapshotRegionProcedure b6da6887b5f5557b8eaf374a87ad833b 2023-10-02T12:37:26,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=81, resume processing ppid=80 2023-10-02T12:37:26,371 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=80, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1696250245378_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:37:26,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=81, ppid=80, state=SUCCESS; SnapshotRegionProcedure b6da6887b5f5557b8eaf374a87ad833b in 563 msec 2023-10-02T12:37:26,372 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=80, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1696250245378_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:37:26,372 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=80, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1696250245378_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:37:26,372 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696250245378_ns2_test-16962502037111 2023-10-02T12:37:26,373 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_1696250245378_ns2_test-16962502037111 2023-10-02T12:37:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=80 2023-10-02T12:37:26,784 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=80, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1696250245378_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:37:26,791 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=80, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1696250245378_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:37:26,792 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/snapshot_1696250245378_ns2_test-16962502037111 to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/snapshot_1696250245378_ns2_test-16962502037111 2023-10-02T12:37:26,793 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=80, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1696250245378_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:37:26,793 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696250245378_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 }, snapshot procedure id = 80 2023-10-02T12:37:26,794 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=80, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=80, snapshot={ ss=snapshot_1696250245378_ns2_test-16962502037111 table=ns2:test-16962502037111 type=FLUSH ttl=0 } in 1.4000 sec 2023-10-02T12:37:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=80 2023-10-02T12:37:27,502 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] backup.TestBackupBase$FullTableBackupClientForTest(226): snapshot copy for backup_1696250240933 2023-10-02T12:37:27,502 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:37:27,504 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-10-02T12:37:27,505 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696250240933_test-1696250203711 2023-10-02T12:37:27,505 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696250243258_default_test-1696250203711 to hdfs://localhost:40817/backupUT/backup_1696250240933/default/test-1696250203711/ 2023-10-02T12:37:27,510 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:37:27,549 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:40817, inputRoot=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09 2023-10-02T12:37:27,549 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,36327,1696250187444_-62506847_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:40817/backupUT/backup_1696250240933/default/test-1696250203711, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:40817/backupUT/backup_1696250240933/default/test-1696250203711/.hbase-snapshot/.tmp/snapshot_1696250243258_default_test-1696250203711 2023-10-02T12:37:27,549 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:40817, inputRoot=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09, snapshotDir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/snapshot_1696250243258_default_test-1696250203711. 2023-10-02T12:37:27,563 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/snapshot_1696250243258_default_test-1696250203711 to hdfs://localhost:40817/backupUT/backup_1696250240933/default/test-1696250203711/.hbase-snapshot/.tmp/snapshot_1696250243258_default_test-1696250203711 2023-10-02T12:37:27,999 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:27,999 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:28,000 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:28,000 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:29,015 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/hadoop-6011139471258887594.jar 2023-10-02T12:37:29,015 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:29,016 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:29,016 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:29,017 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:29,017 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:29,017 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:29,018 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:29,018 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:29,019 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:29,020 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:37:29,020 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:37:29,021 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:37:29,021 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:37:29,022 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:37:29,022 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:37:29,023 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:37:29,023 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:37:29,024 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:37:29,024 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:37:29,024 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:37:29,025 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:37:29,029 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:37:29,029 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:37:29,030 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:37:29,030 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:37:29,030 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:37:29,031 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:37:29,032 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:37:29,116 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696250243258_default_test-1696250203711' hfile list 2023-10-02T12:37:29,125 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-10-02T12:37:29,198 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-10-02T12:37:29,199 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-10-02T12:37:29,349 WARN [Listener at jenkins-hbase3.apache.org/34907 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/mapreduce.cluster.local.dir/1696250249288/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:37:29,349 WARN [Listener at jenkins-hbase3.apache.org/34907 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/mapreduce.cluster.local.dir/1696250249288/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:37:29,535 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:37:29,592 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/default/test-1696250203711/0d8a4463d318b78a2ee236eba02d8a78/f/7287d7a48b5c4307947a2ab13e8623a8 output=hdfs://localhost:40817/backupUT/backup_1696250240933/default/test-1696250203711/archive/data/default/test-1696250203711/0d8a4463d318b78a2ee236eba02d8a78/f/7287d7a48b5c4307947a2ab13e8623a8 2023-10-02T12:37:29,592 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-10-02T12:37:30,432 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:37:30,433 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:37:30,442 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696250243258_default_test-1696250203711 2023-10-02T12:37:30,442 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696250243258_default_test-1696250203711 finished. 2023-10-02T12:37:30,442 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696250240933_ns2:test-16962502037111 2023-10-02T12:37:30,442 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696250245378_ns2_test-16962502037111 to hdfs://localhost:40817/backupUT/backup_1696250240933/ns2/test-16962502037111/ 2023-10-02T12:37:30,443 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:37:30,470 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:40817, inputRoot=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09 2023-10-02T12:37:30,470 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,36327,1696250187444_-1592895631_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:40817/backupUT/backup_1696250240933/ns2/test-16962502037111, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:40817/backupUT/backup_1696250240933/ns2/test-16962502037111/.hbase-snapshot/.tmp/snapshot_1696250245378_ns2_test-16962502037111 2023-10-02T12:37:30,470 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:40817, inputRoot=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09, snapshotDir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/snapshot_1696250245378_ns2_test-16962502037111. 2023-10-02T12:37:30,483 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/snapshot_1696250245378_ns2_test-16962502037111 to hdfs://localhost:40817/backupUT/backup_1696250240933/ns2/test-16962502037111/.hbase-snapshot/.tmp/snapshot_1696250245378_ns2_test-16962502037111 2023-10-02T12:37:30,915 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:30,915 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:30,916 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:30,916 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:31,893 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/hadoop-3663970683327719970.jar 2023-10-02T12:37:31,893 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:31,894 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:31,894 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:31,895 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:31,895 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:31,896 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:31,896 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:31,896 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:31,897 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:37:31,898 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:37:31,898 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:37:31,899 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:37:31,899 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:37:31,900 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:37:31,900 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:37:31,900 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:37:31,901 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:37:31,901 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:37:31,902 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:37:31,902 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:37:31,903 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:37:31,903 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:37:31,904 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:37:31,904 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:37:31,905 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:37:31,905 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:37:31,906 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:37:31,906 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:37:31,953 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696250245378_ns2_test-16962502037111' hfile list 2023-10-02T12:37:31,955 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-10-02T12:37:32,031 WARN [Listener at jenkins-hbase3.apache.org/34907 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/mapreduce.cluster.local.dir/1696250251995/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:37:32,031 WARN [Listener at jenkins-hbase3.apache.org/34907 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/mapreduce.cluster.local.dir/1696250251995/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:37:32,106 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:37:32,133 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns2/test-16962502037111/b6da6887b5f5557b8eaf374a87ad833b/f/a89c04d42b29401c925078c79710cedf output=hdfs://localhost:40817/backupUT/backup_1696250240933/ns2/test-16962502037111/archive/data/ns2/test-16962502037111/b6da6887b5f5557b8eaf374a87ad833b/f/a89c04d42b29401c925078c79710cedf 2023-10-02T12:37:32,133 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-10-02T12:37:33,086 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:37:33,088 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:37:33,095 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696250245378_ns2_test-16962502037111 2023-10-02T12:37:33,095 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696250245378_ns2_test-16962502037111 finished. 2023-10-02T12:37:33,095 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1034): test-1696250203711 2023-10-02T12:37:33,095 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1034): ns2:test-16962502037111 2023-10-02T12:37:33,111 ERROR [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(228): Unexpected BackupException : Failed stage stage_4 in testing java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:245) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.runBackupAndFailAtStage(TestFullBackupWithFailures.java:67) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.testFullBackupWithFailures(TestFullBackupWithFailures.java:55) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:37:33,112 ERROR [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(240): BackupId=backup_1696250240933,startts=1696250243075,failedts=1696250253112,failedphase=SNAPSHOTCOPY,failedmessage=Failed stage stage_4 in testing 2023-10-02T12:37:33,112 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:37:33,112 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696250243258_default_test-1696250203711 2023-10-02T12:37:33,113 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696250243258_default_test-1696250203711" 2023-10-02T12:37:33,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696250243258_default_test-1696250203711 2023-10-02T12:37:33,116 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696250243258_default_test-1696250203711 for backup backup_1696250240933 succeeded. 2023-10-02T12:37:33,116 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696250245378_ns2_test-16962502037111 2023-10-02T12:37:33,117 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696250245378_ns2_test-16962502037111" 2023-10-02T12:37:33,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696250245378_ns2_test-16962502037111 2023-10-02T12:37:33,119 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696250245378_ns2_test-16962502037111 for backup backup_1696250240933 succeeded. 2023-10-02T12:37:33,121 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-10-02T12:37:33,125 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable backup:system 2023-10-02T12:37:33,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-10-02T12:37:33,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=82 2023-10-02T12:37:33,130 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250253129"}]},"ts":"1696250253129"} 2023-10-02T12:37:33,131 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-10-02T12:37:33,133 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-10-02T12:37:33,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, UNASSIGN}] 2023-10-02T12:37:33,135 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=83, ppid=82, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, UNASSIGN 2023-10-02T12:37:33,136 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=83 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:33,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE; CloseRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566}] 2023-10-02T12:37:33,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=82 2023-10-02T12:37:33,289 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] handler.UnassignRegionHandler(119): Close 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:33,289 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1668): Closing 232c4115fe6f97bf9747948e2cf60e81, disabling compactions & flushes 2023-10-02T12:37:33,289 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1690): Closing region backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:33,289 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:33,289 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. after waiting 0 ms 2023-10-02T12:37:33,289 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:33,289 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(2819): Flushing 232c4115fe6f97bf9747948e2cf60e81 2/2 column families, dataSize=999 B heapSize=2.05 KB 2023-10-02T12:37:33,303 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/meta/6b47738021004d718718e6fe8298dc1e is 171, key is trslm:hdfs://localhost:40817/backupUT\x00ns2:test-16962502037111/meta:log-roll-map/1696250253108/Put/seqid=0 2023-10-02T12:37:33,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=82 2023-10-02T12:37:33,708 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=785 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/meta/6b47738021004d718718e6fe8298dc1e 2023-10-02T12:37:33,718 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/ad5e8642b2b747a9a68b26ac07bb740d is 218, key is session:backup_1696250240933/session:context/1696250243075/Put/seqid=0 2023-10-02T12:37:33,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=82 2023-10-02T12:37:34,122 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=214 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/ad5e8642b2b747a9a68b26ac07bb740d 2023-10-02T12:37:34,129 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/meta/6b47738021004d718718e6fe8298dc1e as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/6b47738021004d718718e6fe8298dc1e 2023-10-02T12:37:34,134 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/6b47738021004d718718e6fe8298dc1e, entries=6, sequenceid=72, filesize=5.8 K 2023-10-02T12:37:34,135 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/ad5e8642b2b747a9a68b26ac07bb740d as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/ad5e8642b2b747a9a68b26ac07bb740d 2023-10-02T12:37:34,140 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/ad5e8642b2b747a9a68b26ac07bb740d, entries=1, sequenceid=72, filesize=5.2 K 2023-10-02T12:37:34,141 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(3022): Finished flush of dataSize ~999 B/999, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 232c4115fe6f97bf9747948e2cf60e81 in 852ms, sequenceid=72, compaction requested=true 2023-10-02T12:37:34,154 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/recovered.edits/75.seqid, newMaxSeqId=75, maxSeqId=57 2023-10-02T12:37:34,155 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:37:34,156 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1909): Closed backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:34,156 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] regionserver.HRegion(1622): Region close journal for 232c4115fe6f97bf9747948e2cf60e81: 2023-10-02T12:37:34,157 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=84}] handler.UnassignRegionHandler(163): Closed 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:34,158 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=83 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=CLOSED 2023-10-02T12:37:34,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=84, resume processing ppid=83 2023-10-02T12:37:34,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=84, ppid=83, state=SUCCESS; CloseRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566 in 1.0220 sec 2023-10-02T12:37:34,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=83, resume processing ppid=82 2023-10-02T12:37:34,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=83, ppid=82, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, UNASSIGN in 1.0260 sec 2023-10-02T12:37:34,162 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250254162"}]},"ts":"1696250254162"} 2023-10-02T12:37:34,163 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-10-02T12:37:34,165 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-10-02T12:37:34,167 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=82, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0410 sec 2023-10-02T12:37:34,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=82 2023-10-02T12:37:34,241 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1696250254240 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:37:34,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696250254241 (current time:1696250254241). 2023-10-02T12:37:34,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:37:34,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1696250254240 VERSION not specified, setting to 2 2023-10-02T12:37:34,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8124a6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:37:34,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:37:34,251 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60034, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:37:34,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:34,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:37:34,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=85, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250254240 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:37:34,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250254240 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 85 2023-10-02T12:37:34,255 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=85, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250254240 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:37:34,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-10-02T12:37:34,256 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=85, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250254240 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:37:34,258 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot, current working dir path: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696250254240 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:37:34,260 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=85, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250254240 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:37:34,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-10-02T12:37:34,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-10-02T12:37:34,667 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=85, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250254240 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-10-02T12:37:34,668 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-10-02T12:37:34,668 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-10-02T12:37:34,668 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:34,669 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/6b47738021004d718718e6fe8298dc1e] hfiles 2023-10-02T12:37:34,669 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/6b47738021004d718718e6fe8298dc1e 2023-10-02T12:37:34,669 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:34,671 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/346633fda9724350a60ab080f906d768, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/42d5514f478c436a97d425898f6f3106, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/5dc67774b6954153bced22444bed4827, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/ad5e8642b2b747a9a68b26ac07bb740d] hfiles 2023-10-02T12:37:34,671 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/4): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/346633fda9724350a60ab080f906d768 2023-10-02T12:37:34,671 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/4): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/42d5514f478c436a97d425898f6f3106 2023-10-02T12:37:34,671 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/4): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/5dc67774b6954153bced22444bed4827 2023-10-02T12:37:34,671 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (4/4): hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/ad5e8642b2b747a9a68b26ac07bb740d 2023-10-02T12:37:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-10-02T12:37:35,077 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-10-02T12:37:35,078 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=85, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250254240 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:37:35,078 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=85, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250254240 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:37:35,078 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1696250254240 2023-10-02T12:37:35,079 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696250254240 2023-10-02T12:37:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-10-02T12:37:35,493 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=85, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250254240 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:37:35,502 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=85, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250254240 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:37:35,503 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696250254240 to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1696250254240 2023-10-02T12:37:35,504 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=85, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250254240 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:37:35,504 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250254240 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 85 2023-10-02T12:37:35,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=85, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=85, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696250254240 table=backup:system type=FLUSH ttl=0 } in 1.2520 sec 2023-10-02T12:37:36,128 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 89f0eb5627b232df3a9a7c8d2e1d15a1 changed from -1.0 to 0.0, refreshing cache 2023-10-02T12:37:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=85 2023-10-02T12:37:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=86, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696250240953 type: FLUSH version: 2 ttl: 0 ) 2023-10-02T12:37:36,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-10-02T12:37:36,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=86 2023-10-02T12:37:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=86 2023-10-02T12:37:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=86 2023-10-02T12:37:36,781 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/.tabledesc/.tableinfo.0000000005.635 2023-10-02T12:37:36,782 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/.tabledesc/.tableinfo.0000000006.635 2023-10-02T12:37:36,783 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:37:36,785 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696250240953 type: FLUSH version: 2 ttl: 0 2023-10-02T12:37:36,785 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system 2023-10-02T12:37:36,788 DEBUG [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-10-02T12:37:36,788 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:36,795 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/6b47738021004d718718e6fe8298dc1e to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/archive/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/meta/6b47738021004d718718e6fe8298dc1e 2023-10-02T12:37:36,795 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:36,799 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/ad5e8642b2b747a9a68b26ac07bb740d to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/archive/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/ad5e8642b2b747a9a68b26ac07bb740d 2023-10-02T12:37:36,799 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:36,799 INFO [PEWorker-5 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696250240953 type: FLUSH version: 2 ttl: 0 2023-10-02T12:37:36,800 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696250256800"}]},"ts":"9223372036854775807"} 2023-10-02T12:37:36,802 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:37:36,802 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:37:36,802 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1696250256801"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696250256801"}]},"ts":"1696250256801"} 2023-10-02T12:37:36,804 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:37:36,804 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-10-02T12:37:36,804 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:37:36,804 INFO [PEWorker-5 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-10-02T12:37:36,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=86, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696250240953 type: FLUSH version: 2 ttl: 0 ) in 437 msec 2023-10-02T12:37:36,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=86 2023-10-02T12:37:36,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1696250254240" 2023-10-02T12:37:36,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1696250254240 2023-10-02T12:37:36,979 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.HMaster$11(2631): Client=jenkins//172.31.12.81 enable backup:system 2023-10-02T12:37:36,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] procedure2.ProcedureExecutor(1032): Stored pid=87, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-10-02T12:37:36,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-10-02T12:37:36,982 INFO [PEWorker-4 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-10-02T12:37:36,983 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250256982"}]},"ts":"1696250256982"} 2023-10-02T12:37:36,984 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-10-02T12:37:36,986 DEBUG [PEWorker-4 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-10-02T12:37:36,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN}] 2023-10-02T12:37:36,988 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN 2023-10-02T12:37:36,988 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-10-02T12:37:37,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-10-02T12:37:37,140 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:37,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE; OpenRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566}] 2023-10-02T12:37:37,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-10-02T12:37:37,300 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(130): Open backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:37,300 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7254): Opening region: {ENCODED => 232c4115fe6f97bf9747948e2cf60e81, NAME => 'backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:37:37,300 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:37:37,301 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:37,301 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(888): Instantiated backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:37:37,301 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7296): checking encryption for 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:37,301 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7299): checking classloading for 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:37,302 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:37,303 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 232c4115fe6f97bf9747948e2cf60e81 columnFamilyName meta 2023-10-02T12:37:37,303 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:37,303 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(324): Store=232c4115fe6f97bf9747948e2cf60e81/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:37:37,304 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:37,304 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 232c4115fe6f97bf9747948e2cf60e81 columnFamilyName session 2023-10-02T12:37:37,304 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:37:37,311 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/346633fda9724350a60ab080f906d768 2023-10-02T12:37:37,316 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/42d5514f478c436a97d425898f6f3106 2023-10-02T12:37:37,320 DEBUG [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/5dc67774b6954153bced22444bed4827 2023-10-02T12:37:37,320 INFO [StoreOpener-232c4115fe6f97bf9747948e2cf60e81-1 {}] regionserver.HStore(324): Store=232c4115fe6f97bf9747948e2cf60e81/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:37:37,321 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:37,322 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:37,323 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:37:37,324 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1076): writing seq id for 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:37,325 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): Opened 232c4115fe6f97bf9747948e2cf60e81; next sequenceid=76; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10222789440, jitterRate=-0.04792854189872742}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:37:37,326 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(992): Region open journal for 232c4115fe6f97bf9747948e2cf60e81: 2023-10-02T12:37:37,326 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81., pid=89, masterSystemTime=1696250257293 2023-10-02T12:37:37,328 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.CompactSplit(403): Add compact mark for store 232c4115fe6f97bf9747948e2cf60e81:session, priority=-2147483648, current under compaction store size is 1 2023-10-02T12:37:37,328 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-10-02T12:37:37,328 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2023-10-02T12:37:37,329 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 16482 starting at candidate #0 after considering 1 permutations with 1 in ratio 2023-10-02T12:37:37,329 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] regionserver.HStore(1534): 232c4115fe6f97bf9747948e2cf60e81/session is initiating minor compaction (all files) 2023-10-02T12:37:37,329 INFO [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] regionserver.HRegion(2333): Starting compaction of 232c4115fe6f97bf9747948e2cf60e81/session in backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:37,330 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:37,330 INFO [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] regionserver.HStore(1171): Starting compaction of [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/346633fda9724350a60ab080f906d768, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/42d5514f478c436a97d425898f6f3106, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/5dc67774b6954153bced22444bed4827] into tmpdir=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp, totalSize=16.1 K 2023-10-02T12:37:37,330 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(158): Opened backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:37,330 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=88 updating hbase:meta row=232c4115fe6f97bf9747948e2cf60e81, regionState=OPEN, openSeqNum=76, regionLocation=jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:37,330 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] compactions.Compactor(223): Compacting 346633fda9724350a60ab080f906d768, keycount=3, bloomtype=ROW, size=5.6 K, encoding=NONE, compression=NONE, seqNum=32, earliestPutTs=1696250215658 2023-10-02T12:37:37,331 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] compactions.Compactor(223): Compacting 42d5514f478c436a97d425898f6f3106, keycount=2, bloomtype=ROW, size=5.2 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1696250230126 2023-10-02T12:37:37,331 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] compactions.Compactor(223): Compacting 5dc67774b6954153bced22444bed4827, keycount=2, bloomtype=ROW, size=5.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1696250240830 2023-10-02T12:37:37,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=89, resume processing ppid=88 2023-10-02T12:37:37,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=89, ppid=88, state=SUCCESS; OpenRegionProcedure 232c4115fe6f97bf9747948e2cf60e81, server=jenkins-hbase3.apache.org,37985,1696250191566 in 190 msec 2023-10-02T12:37:37,335 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=88, resume processing ppid=87 2023-10-02T12:37:37,335 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=88, ppid=87, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=232c4115fe6f97bf9747948e2cf60e81, ASSIGN in 347 msec 2023-10-02T12:37:37,336 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696250257335"}]},"ts":"1696250257335"} 2023-10-02T12:37:37,337 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-10-02T12:37:37,339 INFO [PEWorker-3 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-10-02T12:37:37,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=87, state=SUCCESS; EnableTableProcedure table=backup:system in 361 msec 2023-10-02T12:37:37,349 INFO [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 232c4115fe6f97bf9747948e2cf60e81#session#compaction#17 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2023-10-02T12:37:37,350 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/56466c6d0c4944ab92b2f21e769b54bd is 355, key is session:backup_1696250230252/session:context/1696250240830/Put/seqid=0 2023-10-02T12:37:37,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=87 2023-10-02T12:37:37,585 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-10-02T12:37:37,585 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:37:37,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:37:37,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36327 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:37:37,593 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:37:37,593 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(184): Trying to cleanup up target dir. Current backup phase: SNAPSHOTCOPY 2023-10-02T12:37:37,594 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:40817/backupUT/backup_1696250240933/default/test-1696250203711 done. 2023-10-02T12:37:37,595 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(208): hdfs://localhost:40817/backupUT/backup_1696250240933/default is empty, remove it. 2023-10-02T12:37:37,596 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(198): Cleaning up uncompleted backup data at hdfs://localhost:40817/backupUT/backup_1696250240933/ns2/test-16962502037111 done. 2023-10-02T12:37:37,596 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(208): hdfs://localhost:40817/backupUT/backup_1696250240933/ns2 is empty, remove it. 2023-10-02T12:37:37,598 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:37:37,603 ERROR [Listener at jenkins-hbase3.apache.org/34907 {}] impl.TableBackupClient(246): Backup backup_1696250240933 failed. Backup session finished. Status: FAILURE 2023-10-02T12:37:37,604 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:37,604 ERROR [Listener at jenkins-hbase3.apache.org/34907 {}] backup.BackupDriver(201): Error running command-line tool java.io.IOException: java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:255) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.backupTables(BackupAdminImpl.java:603) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupCommands$CreateCommand.execute(BackupCommands.java:345) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.parseAndRun(BackupDriver.java:134) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.doWork(BackupDriver.java:169) ~[classes/:?] at org.apache.hadoop.hbase.backup.BackupDriver.run(BackupDriver.java:199) ~[classes/:?] at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.runBackupAndFailAtStage(TestFullBackupWithFailures.java:67) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestFullBackupWithFailures.testFullBackupWithFailures(TestFullBackupWithFailures.java:55) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: Failed stage stage_4 in testing at org.apache.hadoop.hbase.backup.impl.TableBackupClient.failStageIf(TableBackupClient.java:392) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupBase$FullTableBackupClientForTest.execute(TestBackupBase.java:245) ~[test-classes/:?] ... 32 more 2023-10-02T12:37:37,680 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] hbase.ResourceChecker(175): after: backup.TestFullBackupWithFailures#testFullBackupWithFailures Thread=605 (was 584) Potentially hanging thread: rs(jenkins-hbase3.apache.org,37985,1696250191566)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57296@0x611c365e java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$486/746154966.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-388412218-172.31.12.81-1696250182869:blk_1073741907_1083, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataStreamer for file /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/56466c6d0c4944ab92b2f21e769b54bd java.lang.Object.wait(Native Method) org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:678) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57296@0x611c365e-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: RS:0;jenkins-hbase3:37985-shortCompactions-0 java.lang.Thread.sleep(Native Method) org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:940) org.apache.hadoop.hdfs.DFSOutputStream.completeFile(DFSOutputStream.java:880) org.apache.hadoop.hdfs.DFSOutputStream.closeImpl(DFSOutputStream.java:863) org.apache.hadoop.hdfs.DFSOutputStream.close(DFSOutputStream.java:818) org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72) org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.finishClose(HFileWriterImpl.java:858) org.apache.hadoop.hbase.io.hfile.HFileWriterImpl.close(HFileWriterImpl.java:679) org.apache.hadoop.hbase.regionserver.StoreFileWriter.close(StoreFileWriter.java:377) org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:72) org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.commitWriter(DefaultCompactor.java:40) org.apache.hadoop.hbase.regionserver.compactions.Compactor.compact(Compactor.java:396) org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor.compact(DefaultCompactor.java:64) org.apache.hadoop.hbase.regionserver.DefaultStoreEngine$DefaultCompactionContext.compact(DefaultStoreEngine.java:122) org.apache.hadoop.hbase.regionserver.HStore.compact(HStore.java:1176) org.apache.hadoop.hbase.regionserver.HRegion.compact(HRegion.java:2341) org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.doCompaction(CompactSplit.java:667) org.apache.hadoop.hbase.regionserver.CompactSplit$CompactionRunner.run(CompactSplit.java:716) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1149368692_16 at /127.0.0.1:56730 [Waiting for operation #20] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1500a07e-shared-pool-34 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x70daa3a4-shared-pool-35 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1500a07e-shared-pool-33 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x70daa3a4-shared-pool-34 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1149368692_16 at /127.0.0.1:40870 [Waiting for operation #6] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/cluster_523362f7-c993-0d07-9ba7-304dd19a6089/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-885745764_16 at /127.0.0.1:56662 [Receiving block BP-388412218-172.31.12.81-1696250182869:blk_1073741908_1084] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-388412218-172.31.12.81-1696250182869:blk_1073741908_1084, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1149368692_16 at /127.0.0.1:40886 [Waiting for operation #7] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1500a07e-shared-pool-37 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57296@0x611c365e-SendThread(127.0.0.1:57296) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:228) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-885745764_16 at /127.0.0.1:56654 [Receiving block BP-388412218-172.31.12.81-1696250182869:blk_1073741907_1083] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x70daa3a4-shared-pool-31 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x70daa3a4-shared-pool-30 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1149368692_16 at /127.0.0.1:56648 [Waiting for operation #33] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1500a07e-shared-pool-35 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1500a07e-shared-pool-32 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: region-location-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081) java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1500a07e-shared-pool-36 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x70daa3a4-shared-pool-32 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,37985,1696250191566)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,37985,1696250191566)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,36327,1696250187444)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,37985,1696250191566' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/cluster_523362f7-c993-0d07-9ba7-304dd19a6089/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x70daa3a4-shared-pool-33 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=677 (was 660) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=379 (was 513), ProcessCount=181 (was 181), AvailableMemoryMB=4463 (was 5038) 2023-10-02T12:37:37,680 WARN [Listener at jenkins-hbase3.apache.org/34907 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-10-02T12:37:37,688 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-10-02T12:37:37,688 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:37,688 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-10-02T12:37:37,688 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] util.JVMClusterUtil(257): Found active master hash=305517833, stopped=false 2023-10-02T12:37:37,689 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,36327,1696250187444 2023-10-02T12:37:37,691 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] procedure2.ProcedureExecutor(630): Stopping 2023-10-02T12:37:37,692 DEBUG [Listener at jenkins-hbase3.apache.org/34907 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:37,692 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,37985,1696250191566' ***** 2023-10-02T12:37:37,692 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-10-02T12:37:37,693 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-10-02T12:37:37,709 INFO [RS:0;jenkins-hbase3:37985 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@33da7094{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-10-02T12:37:37,715 INFO [RS:0;jenkins-hbase3:37985 {}] server.AbstractConnector(383): Stopped ServerConnector@6e35d418{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:37:37,715 INFO [RS:0;jenkins-hbase3:37985 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-10-02T12:37:37,716 INFO [RS:0;jenkins-hbase3:37985 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@9fbde29{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-10-02T12:37:37,717 INFO [RS:0;jenkins-hbase3:37985 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@7e8f64e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/hadoop.log.dir/,STOPPED} 2023-10-02T12:37:37,719 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.HeapMemoryManager(220): Stopping 2023-10-02T12:37:37,719 INFO [RS:0;jenkins-hbase3:37985 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-10-02T12:37:37,720 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-10-02T12:37:37,720 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-10-02T12:37:37,720 INFO [RS:0;jenkins-hbase3:37985 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-10-02T12:37:37,720 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(3528): Received CLOSE for 0d8a4463d318b78a2ee236eba02d8a78 2023-10-02T12:37:37,720 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(3528): Received CLOSE for 89f0eb5627b232df3a9a7c8d2e1d15a1 2023-10-02T12:37:37,720 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(3528): Received CLOSE for 08d5f9946b11fd88451ddcf204fe133f 2023-10-02T12:37:37,720 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(3528): Received CLOSE for 1668fd8ee9c956d5c525bdda4f6f5b90 2023-10-02T12:37:37,720 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(3528): Received CLOSE for 232c4115fe6f97bf9747948e2cf60e81 2023-10-02T12:37:37,720 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(3528): Received CLOSE for d15e6e23ef0ca9fe47b3a30aa65c34b4 2023-10-02T12:37:37,720 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(3528): Received CLOSE for b6da6887b5f5557b8eaf374a87ad833b 2023-10-02T12:37:37,720 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,37985,1696250191566 2023-10-02T12:37:37,720 DEBUG [RS:0;jenkins-hbase3:37985 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:37,721 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-10-02T12:37:37,721 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-10-02T12:37:37,721 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 0d8a4463d318b78a2ee236eba02d8a78, disabling compactions & flushes 2023-10-02T12:37:37,721 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. 2023-10-02T12:37:37,721 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. 2023-10-02T12:37:37,721 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. after waiting 0 ms 2023-10-02T12:37:37,721 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. 2023-10-02T12:37:37,725 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/default/test-1696250203711/0d8a4463d318b78a2ee236eba02d8a78/recovered.edits/107.seqid, newMaxSeqId=107, maxSeqId=1 2023-10-02T12:37:37,725 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:37:37,726 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. 2023-10-02T12:37:37,726 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 0d8a4463d318b78a2ee236eba02d8a78: 2023-10-02T12:37:37,726 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1696250203711,,1696250204786.0d8a4463d318b78a2ee236eba02d8a78. 2023-10-02T12:37:37,727 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 89f0eb5627b232df3a9a7c8d2e1d15a1, disabling compactions & flushes 2023-10-02T12:37:37,727 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1. 2023-10-02T12:37:37,727 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1. 2023-10-02T12:37:37,727 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1. after waiting 0 ms 2023-10-02T12:37:37,727 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1. 2023-10-02T12:37:37,727 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 89f0eb5627b232df3a9a7c8d2e1d15a1 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-10-02T12:37:37,741 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/namespace/89f0eb5627b232df3a9a7c8d2e1d15a1/.tmp/info/8e26a0ea7e114c74a6090023345c44a6 is 45, key is default/info:d/1696250195707/Put/seqid=0 2023-10-02T12:37:37,761 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/56466c6d0c4944ab92b2f21e769b54bd as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/56466c6d0c4944ab92b2f21e769b54bd 2023-10-02T12:37:37,767 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-10-02T12:37:37,768 INFO [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] regionserver.HStore(1331): Completed compaction of 3 (all) file(s) in 232c4115fe6f97bf9747948e2cf60e81/session of 232c4115fe6f97bf9747948e2cf60e81 into 56466c6d0c4944ab92b2f21e769b54bd(size=6.2 K), total size for store is 6.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2023-10-02T12:37:37,768 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] regionserver.HRegion(2363): Compaction status journal for 232c4115fe6f97bf9747948e2cf60e81: 2023-10-02T12:37:37,768 INFO [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81., storeName=232c4115fe6f97bf9747948e2cf60e81/session, priority=13, startTime=1696250257326; duration=0sec 2023-10-02T12:37:37,768 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2023-10-02T12:37:37,768 DEBUG [RS:0;jenkins-hbase3:37985-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 232c4115fe6f97bf9747948e2cf60e81:session 2023-10-02T12:37:37,768 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-10-02T12:37:37,769 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-10-02T12:37:37,769 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(1579): Waiting on 7 regions to close 2023-10-02T12:37:37,769 DEBUG [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(1583): Online Regions={89f0eb5627b232df3a9a7c8d2e1d15a1=hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1., 1588230740=hbase:meta,,1.1588230740, 08d5f9946b11fd88451ddcf204fe133f=ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f., 1668fd8ee9c956d5c525bdda4f6f5b90=ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90., 232c4115fe6f97bf9747948e2cf60e81=backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81., d15e6e23ef0ca9fe47b3a30aa65c34b4=backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4., b6da6887b5f5557b8eaf374a87ad833b=ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b.} 2023-10-02T12:37:37,770 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-10-02T12:37:37,770 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-10-02T12:37:37,770 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-10-02T12:37:37,770 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-10-02T12:37:37,770 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-10-02T12:37:37,770 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=19.18 KB heapSize=33.47 KB 2023-10-02T12:37:37,771 DEBUG [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(1609): Waiting on 08d5f9946b11fd88451ddcf204fe133f, 1588230740, 1668fd8ee9c956d5c525bdda4f6f5b90, 232c4115fe6f97bf9747948e2cf60e81, 89f0eb5627b232df3a9a7c8d2e1d15a1, b6da6887b5f5557b8eaf374a87ad833b, d15e6e23ef0ca9fe47b3a30aa65c34b4 2023-10-02T12:37:37,782 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-10-02T12:37:37,793 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/meta/1588230740/.tmp/info/0a23fa3c295d4f0581d47168d7cbd037 is 159, key is ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b./info:regioninfo/1696250206096/Put/seqid=0 2023-10-02T12:37:37,971 DEBUG [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(1609): Waiting on 08d5f9946b11fd88451ddcf204fe133f, 1588230740, 1668fd8ee9c956d5c525bdda4f6f5b90, 232c4115fe6f97bf9747948e2cf60e81, 89f0eb5627b232df3a9a7c8d2e1d15a1, b6da6887b5f5557b8eaf374a87ad833b, d15e6e23ef0ca9fe47b3a30aa65c34b4 2023-10-02T12:37:38,147 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/namespace/89f0eb5627b232df3a9a7c8d2e1d15a1/.tmp/info/8e26a0ea7e114c74a6090023345c44a6 2023-10-02T12:37:38,171 DEBUG [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(1609): Waiting on 08d5f9946b11fd88451ddcf204fe133f, 1588230740, 1668fd8ee9c956d5c525bdda4f6f5b90, 232c4115fe6f97bf9747948e2cf60e81, 89f0eb5627b232df3a9a7c8d2e1d15a1, b6da6887b5f5557b8eaf374a87ad833b, d15e6e23ef0ca9fe47b3a30aa65c34b4 2023-10-02T12:37:38,172 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/namespace/89f0eb5627b232df3a9a7c8d2e1d15a1/.tmp/info/8e26a0ea7e114c74a6090023345c44a6 as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/namespace/89f0eb5627b232df3a9a7c8d2e1d15a1/info/8e26a0ea7e114c74a6090023345c44a6 2023-10-02T12:37:38,182 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/namespace/89f0eb5627b232df3a9a7c8d2e1d15a1/info/8e26a0ea7e114c74a6090023345c44a6, entries=7, sequenceid=11, filesize=5.1 K 2023-10-02T12:37:38,184 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 89f0eb5627b232df3a9a7c8d2e1d15a1 in 456ms, sequenceid=11, compaction requested=false 2023-10-02T12:37:38,205 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.61 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/meta/1588230740/.tmp/info/0a23fa3c295d4f0581d47168d7cbd037 2023-10-02T12:37:38,269 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/namespace/89f0eb5627b232df3a9a7c8d2e1d15a1/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-10-02T12:37:38,270 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:37:38,272 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1. 2023-10-02T12:37:38,273 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 89f0eb5627b232df3a9a7c8d2e1d15a1: 2023-10-02T12:37:38,273 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1696250194722.89f0eb5627b232df3a9a7c8d2e1d15a1. 2023-10-02T12:37:38,274 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 08d5f9946b11fd88451ddcf204fe133f, disabling compactions & flushes 2023-10-02T12:37:38,274 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f. 2023-10-02T12:37:38,274 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f. 2023-10-02T12:37:38,274 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f. after waiting 0 ms 2023-10-02T12:37:38,274 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f. 2023-10-02T12:37:38,303 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/meta/1588230740/.tmp/table/eba2e038a06041ebb807d8fd60d6afda is 59, key is ns2:test-16962502037111/table:state/1696250206110/Put/seqid=0 2023-10-02T12:37:38,326 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns4/test-16962502037113/08d5f9946b11fd88451ddcf204fe133f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-10-02T12:37:38,341 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:37:38,345 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f. 2023-10-02T12:37:38,345 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 08d5f9946b11fd88451ddcf204fe133f: 2023-10-02T12:37:38,345 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16962502037113,,1696250207020.08d5f9946b11fd88451ddcf204fe133f. 2023-10-02T12:37:38,350 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 1668fd8ee9c956d5c525bdda4f6f5b90, disabling compactions & flushes 2023-10-02T12:37:38,350 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90. 2023-10-02T12:37:38,350 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90. 2023-10-02T12:37:38,350 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90. after waiting 0 ms 2023-10-02T12:37:38,350 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90. 2023-10-02T12:37:38,362 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns3/test-16962502037112/1668fd8ee9c956d5c525bdda4f6f5b90/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-10-02T12:37:38,363 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:37:38,370 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90. 2023-10-02T12:37:38,370 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 1668fd8ee9c956d5c525bdda4f6f5b90: 2023-10-02T12:37:38,370 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16962502037112,,1696250206390.1668fd8ee9c956d5c525bdda4f6f5b90. 2023-10-02T12:37:38,370 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 232c4115fe6f97bf9747948e2cf60e81, disabling compactions & flushes 2023-10-02T12:37:38,370 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:38,370 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:38,370 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. after waiting 0 ms 2023-10-02T12:37:38,370 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:38,370 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 232c4115fe6f97bf9747948e2cf60e81 2/2 column families, dataSize=395 B heapSize=1.03 KB 2023-10-02T12:37:38,374 DEBUG [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 232c4115fe6f97bf9747948e2cf60e81, b6da6887b5f5557b8eaf374a87ad833b, d15e6e23ef0ca9fe47b3a30aa65c34b4 2023-10-02T12:37:38,387 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/df3e2666997145ae8ddebd422b2ddde1 is 355, key is session:backup_1696250240933/session:context/1696250257597/Put/seqid=0 2023-10-02T12:37:38,430 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=395 B at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/df3e2666997145ae8ddebd422b2ddde1 2023-10-02T12:37:38,446 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/.tmp/session/df3e2666997145ae8ddebd422b2ddde1 as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/df3e2666997145ae8ddebd422b2ddde1 2023-10-02T12:37:38,467 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/df3e2666997145ae8ddebd422b2ddde1, entries=2, sequenceid=81, filesize=5.3 K 2023-10-02T12:37:38,471 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~395 B/395, heapSize ~784 B/784, currentSize=0 B/0 for 232c4115fe6f97bf9747948e2cf60e81 in 101ms, sequenceid=81, compaction requested=false 2023-10-02T12:37:38,487 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-10-02T12:37:38,487 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-10-02T12:37:38,497 DEBUG [StoreCloser-backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.-1 {}] regionserver.HStore(2307): Moving the files [hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/346633fda9724350a60ab080f906d768, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/42d5514f478c436a97d425898f6f3106, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/5dc67774b6954153bced22444bed4827] to archive 2023-10-02T12:37:38,506 DEBUG [StoreCloser-backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2023-10-02T12:37:38,515 DEBUG [StoreCloser-backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/346633fda9724350a60ab080f906d768 to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/archive/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/346633fda9724350a60ab080f906d768 2023-10-02T12:37:38,517 DEBUG [StoreCloser-backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/42d5514f478c436a97d425898f6f3106 to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/archive/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/42d5514f478c436a97d425898f6f3106 2023-10-02T12:37:38,520 DEBUG [StoreCloser-backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/5dc67774b6954153bced22444bed4827 to hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/archive/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/session/5dc67774b6954153bced22444bed4827 2023-10-02T12:37:38,538 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system/232c4115fe6f97bf9747948e2cf60e81/recovered.edits/84.seqid, newMaxSeqId=84, maxSeqId=75 2023-10-02T12:37:38,539 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:37:38,539 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:38,539 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 232c4115fe6f97bf9747948e2cf60e81: 2023-10-02T12:37:38,539 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1696250207893.232c4115fe6f97bf9747948e2cf60e81. 2023-10-02T12:37:38,540 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing d15e6e23ef0ca9fe47b3a30aa65c34b4, disabling compactions & flushes 2023-10-02T12:37:38,540 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4. 2023-10-02T12:37:38,541 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4. 2023-10-02T12:37:38,541 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4. after waiting 0 ms 2023-10-02T12:37:38,541 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4. 2023-10-02T12:37:38,545 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/backup/system_bulk/d15e6e23ef0ca9fe47b3a30aa65c34b4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-10-02T12:37:38,546 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:37:38,547 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4. 2023-10-02T12:37:38,547 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for d15e6e23ef0ca9fe47b3a30aa65c34b4: 2023-10-02T12:37:38,547 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1696250209050.d15e6e23ef0ca9fe47b3a30aa65c34b4. 2023-10-02T12:37:38,548 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing b6da6887b5f5557b8eaf374a87ad833b, disabling compactions & flushes 2023-10-02T12:37:38,548 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. 2023-10-02T12:37:38,548 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. 2023-10-02T12:37:38,548 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. after waiting 0 ms 2023-10-02T12:37:38,548 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. 2023-10-02T12:37:38,574 DEBUG [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, b6da6887b5f5557b8eaf374a87ad833b 2023-10-02T12:37:38,596 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/ns2/test-16962502037111/b6da6887b5f5557b8eaf374a87ad833b/recovered.edits/107.seqid, newMaxSeqId=107, maxSeqId=1 2023-10-02T12:37:38,596 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:37:38,598 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. 2023-10-02T12:37:38,598 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for b6da6887b5f5557b8eaf374a87ad833b: 2023-10-02T12:37:38,598 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16962502037111,,1696250205589.b6da6887b5f5557b8eaf374a87ad833b. 2023-10-02T12:37:38,758 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.58 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/meta/1588230740/.tmp/table/eba2e038a06041ebb807d8fd60d6afda 2023-10-02T12:37:38,767 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/meta/1588230740/.tmp/info/0a23fa3c295d4f0581d47168d7cbd037 as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/meta/1588230740/info/0a23fa3c295d4f0581d47168d7cbd037 2023-10-02T12:37:38,774 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(1579): Waiting on 1 regions to close 2023-10-02T12:37:38,775 DEBUG [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(1583): Online Regions={1588230740=hbase:meta,,1.1588230740} 2023-10-02T12:37:38,775 DEBUG [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(1609): Waiting on 1588230740 2023-10-02T12:37:38,776 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/meta/1588230740/info/0a23fa3c295d4f0581d47168d7cbd037, entries=75, sequenceid=89, filesize=14.5 K 2023-10-02T12:37:38,778 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/meta/1588230740/.tmp/table/eba2e038a06041ebb807d8fd60d6afda as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/meta/1588230740/table/eba2e038a06041ebb807d8fd60d6afda 2023-10-02T12:37:38,788 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/meta/1588230740/table/eba2e038a06041ebb807d8fd60d6afda, entries=15, sequenceid=89, filesize=5.8 K 2023-10-02T12:37:38,789 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~19.18 KB/19644, heapSize ~33.19 KB/33984, currentSize=0 B/0 for 1588230740 in 1019ms, sequenceid=89, compaction requested=false 2023-10-02T12:37:38,897 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/data/hbase/meta/1588230740/recovered.edits/92.seqid, newMaxSeqId=92, maxSeqId=1 2023-10-02T12:37:38,898 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:37:38,898 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-02T12:37:38,899 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-10-02T12:37:38,899 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-10-02T12:37:38,899 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-10-02T12:37:38,975 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,37985,1696250191566; all regions closed. 2023-10-02T12:37:39,008 DEBUG [RS:0;jenkins-hbase3:37985 {}] wal.AbstractFSWAL(1062): Moved 4 WAL file(s) to /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/oldWALs 2023-10-02T12:37:39,008 INFO [RS:0;jenkins-hbase3:37985 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C37985%2C1696250191566.meta:.meta(num 1696250243092) 2023-10-02T12:37:39,014 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/WALs/jenkins-hbase3.apache.org,37985,1696250191566/jenkins-hbase3.apache.org%2C37985%2C1696250191566.1696250243110 not finished, retry = 0 2023-10-02T12:37:39,120 DEBUG [RS:0;jenkins-hbase3:37985 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/oldWALs 2023-10-02T12:37:39,120 INFO [RS:0;jenkins-hbase3:37985 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C37985%2C1696250191566:(num 1696250243110) 2023-10-02T12:37:39,120 DEBUG [RS:0;jenkins-hbase3:37985 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:39,120 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.LeaseManager(133): Closed leases 2023-10-02T12:37:39,121 INFO [RS:0;jenkins-hbase3:37985 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-10-02T12:37:39,121 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-10-02T12:37:39,124 INFO [RS:0;jenkins-hbase3:37985 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:37985 2023-10-02T12:37:39,136 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,37985,1696250191566] 2023-10-02T12:37:39,136 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,37985,1696250191566; numProcessing=1 2023-10-02T12:37:39,137 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,37985,1696250191566 expired; onlineServers=0 2023-10-02T12:37:39,137 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,36327,1696250187444' ***** 2023-10-02T12:37:39,137 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-10-02T12:37:39,139 DEBUG [M:0;jenkins-hbase3:36327 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@255750ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-10-02T12:37:39,139 INFO [M:0;jenkins-hbase3:36327 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-10-02T12:37:39,223 INFO [M:0;jenkins-hbase3:36327 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@2afa8b4f{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-10-02T12:37:39,224 INFO [M:0;jenkins-hbase3:36327 {}] server.AbstractConnector(383): Stopped ServerConnector@5869c901{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:37:39,224 INFO [M:0;jenkins-hbase3:36327 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-10-02T12:37:39,234 INFO [M:0;jenkins-hbase3:36327 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@597b40c3{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-10-02T12:37:39,239 INFO [M:0;jenkins-hbase3:36327 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@25efe20a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/hadoop.log.dir/,STOPPED} 2023-10-02T12:37:39,246 INFO [M:0;jenkins-hbase3:36327 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,36327,1696250187444 2023-10-02T12:37:39,246 INFO [M:0;jenkins-hbase3:36327 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,36327,1696250187444; all regions closed. 2023-10-02T12:37:39,246 DEBUG [M:0;jenkins-hbase3:36327 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:39,246 INFO [M:0;jenkins-hbase3:36327 {}] master.HMaster(1665): Stopping master jetty server 2023-10-02T12:37:39,248 INFO [RS:0;jenkins-hbase3:37985 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,37985,1696250191566; zookeeper connection closed. 2023-10-02T12:37:39,249 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7ba1b112 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7ba1b112 2023-10-02T12:37:39,249 INFO [M:0;jenkins-hbase3:36327 {}] server.AbstractConnector(383): Stopped ServerConnector@26fbbebc{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:37:39,250 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-10-02T12:37:39,254 DEBUG [M:0;jenkins-hbase3:36327 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-10-02T12:37:39,254 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-10-02T12:37:39,255 INFO [M:0;jenkins-hbase3:36327 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-10-02T12:37:39,255 DEBUG [M:0;jenkins-hbase3:36327 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:37:39,255 DEBUG [M:0;jenkins-hbase3:36327 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-10-02T12:37:39,255 INFO [M:0;jenkins-hbase3:36327 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-10-02T12:37:39,255 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696250193339 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696250193339,5,FailOnTimeoutGroup] 2023-10-02T12:37:39,255 DEBUG [M:0;jenkins-hbase3:36327 {}] master.HMaster(1688): Stopping service threads 2023-10-02T12:37:39,256 INFO [M:0;jenkins-hbase3:36327 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-10-02T12:37:39,256 ERROR [M:0;jenkins-hbase3:36327 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-10-02T12:37:39,257 INFO [M:0;jenkins-hbase3:36327 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-10-02T12:37:39,257 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-10-02T12:37:39,258 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696250193342 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696250193342,5,FailOnTimeoutGroup] 2023-10-02T12:37:39,262 WARN [M:0;jenkins-hbase3:36327 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-10-02T12:37:39,262 INFO [M:0;jenkins-hbase3:36327 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-10-02T12:37:39,262 INFO [M:0;jenkins-hbase3:36327 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-10-02T12:37:39,267 DEBUG [M:0;jenkins-hbase3:36327 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-10-02T12:37:39,281 INFO [M:0;jenkins-hbase3:36327 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:37:39,281 DEBUG [M:0;jenkins-hbase3:36327 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:37:39,281 DEBUG [M:0;jenkins-hbase3:36327 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-10-02T12:37:39,281 DEBUG [M:0;jenkins-hbase3:36327 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:37:39,281 INFO [M:0;jenkins-hbase3:36327 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=336.64 KB heapSize=405.53 KB 2023-10-02T12:37:39,294 DEBUG [M:0;jenkins-hbase3:36327 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5f6f411db6ed4b048f90fd03b8b784f9 is 88, key is hbase:meta,,1/info:sn/1696250193990/Put/seqid=0 2023-10-02T12:37:39,700 INFO [M:0;jenkins-hbase3:36327 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=824 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5f6f411db6ed4b048f90fd03b8b784f9 2023-10-02T12:37:39,731 DEBUG [M:0;jenkins-hbase3:36327 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2b4986fe08654cf989e74632255d020f is 1038, key is \x00\x00\x00\x00\x00\x00\x00&/proc:d/1696250214775/Put/seqid=0 2023-10-02T12:37:39,750 INFO [M:0;jenkins-hbase3:36327 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=336.04 KB at sequenceid=824 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2b4986fe08654cf989e74632255d020f 2023-10-02T12:37:39,758 INFO [M:0;jenkins-hbase3:36327 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2b4986fe08654cf989e74632255d020f 2023-10-02T12:37:39,778 DEBUG [M:0;jenkins-hbase3:36327 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/055b2d39f3204ac99f9986ceff40c0ed is 82, key is jenkins-hbase3.apache.org,37985,1696250191566/rs:state/1696250193360/Put/seqid=0 2023-10-02T12:37:39,778 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:37:39,832 INFO [M:0;jenkins-hbase3:36327 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=824 (bloomFilter=true), to=hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/055b2d39f3204ac99f9986ceff40c0ed 2023-10-02T12:37:39,833 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16962502037111 2023-10-02T12:37:39,833 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-02T12:37:39,833 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-10-02T12:37:39,833 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-10-02T12:37:39,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-10-02T12:37:39,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16962502037112 2023-10-02T12:37:39,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-10-02T12:37:39,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1696250203711 2023-10-02T12:37:39,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16962502037113 2023-10-02T12:37:39,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:37:39,870 DEBUG [M:0;jenkins-hbase3:36327 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5f6f411db6ed4b048f90fd03b8b784f9 as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5f6f411db6ed4b048f90fd03b8b784f9 2023-10-02T12:37:39,899 INFO [M:0;jenkins-hbase3:36327 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5f6f411db6ed4b048f90fd03b8b784f9, entries=8, sequenceid=824, filesize=5.5 K 2023-10-02T12:37:39,914 DEBUG [M:0;jenkins-hbase3:36327 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2b4986fe08654cf989e74632255d020f as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2b4986fe08654cf989e74632255d020f 2023-10-02T12:37:39,951 INFO [M:0;jenkins-hbase3:36327 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2b4986fe08654cf989e74632255d020f 2023-10-02T12:37:39,951 INFO [M:0;jenkins-hbase3:36327 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2b4986fe08654cf989e74632255d020f, entries=89, sequenceid=824, filesize=26.7 K 2023-10-02T12:37:39,968 DEBUG [M:0;jenkins-hbase3:36327 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/055b2d39f3204ac99f9986ceff40c0ed as hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/055b2d39f3204ac99f9986ceff40c0ed 2023-10-02T12:37:40,008 INFO [M:0;jenkins-hbase3:36327 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:40817/user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/055b2d39f3204ac99f9986ceff40c0ed, entries=1, sequenceid=824, filesize=5.1 K 2023-10-02T12:37:40,013 INFO [M:0;jenkins-hbase3:36327 {}] regionserver.HRegion(3022): Finished flush of dataSize ~336.64 KB/344715, heapSize ~405.23 KB/414960, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 731ms, sequenceid=824, compaction requested=false 2023-10-02T12:37:40,023 INFO [M:0;jenkins-hbase3:36327 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:37:40,023 DEBUG [M:0;jenkins-hbase3:36327 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:37:40,054 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/d8ea2498-e527-32d2-7803-82f142c58f09/MasterData/WALs/jenkins-hbase3.apache.org,36327,1696250187444/jenkins-hbase3.apache.org%2C36327%2C1696250187444.1696250192492 not finished, retry = 0 2023-10-02T12:37:40,156 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-10-02T12:37:40,156 INFO [M:0;jenkins-hbase3:36327 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-10-02T12:37:40,156 INFO [M:0;jenkins-hbase3:36327 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-10-02T12:37:40,157 INFO [M:0;jenkins-hbase3:36327 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:36327 2023-10-02T12:37:40,279 INFO [M:0;jenkins-hbase3:36327 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,36327,1696250187444; zookeeper connection closed. 2023-10-02T12:37:40,282 WARN [Listener at jenkins-hbase3.apache.org/34907 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-10-02T12:37:40,355 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-10-02T12:37:40,465 WARN [BP-388412218-172.31.12.81-1696250182869 heartbeating to localhost/127.0.0.1:40817 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-10-02T12:37:40,466 WARN [BP-388412218-172.31.12.81-1696250182869 heartbeating to localhost/127.0.0.1:40817 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-388412218-172.31.12.81-1696250182869 (Datanode Uuid 0957471b-e404-4b45-bd6c-b720920e99f1) service to localhost/127.0.0.1:40817 2023-10-02T12:37:40,468 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/cluster_523362f7-c993-0d07-9ba7-304dd19a6089/dfs/data/data1/current/BP-388412218-172.31.12.81-1696250182869 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-02T12:37:40,469 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/c29c109d-8984-3649-a9b0-b61e02313737/cluster_523362f7-c993-0d07-9ba7-304dd19a6089/dfs/data/data2/current/BP-388412218-172.31.12.81-1696250182869 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-02T12:37:40,515 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-10-02T12:37:40,693 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-10-02T12:37:40,693 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-10-02T12:37:40,711 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:37:40,833 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:37:40,950 ERROR [Thread[Thread-276,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:37:40,951 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:37:41,057 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-10-02T12:37:41,063 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-10-02T12:37:41,065 ERROR [Thread[Thread-286,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:37:41,071 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:37:41,174 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:37:41,175 INFO [Listener at jenkins-hbase3.apache.org/34907 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped