2023-10-02 12:24:32,725 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@776aec5c 2023-10-02 12:24:32,744 main DEBUG Took 0.016288 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-10-02 12:24:32,745 main DEBUG PluginManager 'Core' found 129 plugins 2023-10-02 12:24:32,746 main DEBUG PluginManager 'Level' found 0 plugins 2023-10-02 12:24:32,747 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-10-02 12:24:32,749 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:24:32,761 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-10-02 12:24:32,793 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:24:32,795 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:24:32,796 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:24:32,796 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:24:32,797 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:24:32,797 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:24:32,798 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:24:32,798 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:24:32,798 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:24:32,799 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:24:32,799 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:24:32,800 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:24:32,800 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:24:32,800 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:24:32,801 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:24:32,801 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:24:32,802 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:24:32,802 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:24:32,803 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:24:32,803 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:24:32,803 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:24:32,804 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:24:32,804 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:24:32,805 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:24:32,805 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:24:32,806 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-10-02 12:24:32,808 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:24:32,811 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-10-02 12:24:32,813 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-10-02 12:24:32,814 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-10-02 12:24:32,816 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-10-02 12:24:32,816 main DEBUG PluginManager 'Converter' found 47 plugins 2023-10-02 12:24:32,833 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-10-02 12:24:32,836 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-10-02 12:24:32,838 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-10-02 12:24:32,838 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-10-02 12:24:32,839 main DEBUG createAppenders(={Console}) 2023-10-02 12:24:32,840 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@776aec5c initialized 2023-10-02 12:24:32,840 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@776aec5c 2023-10-02 12:24:32,840 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@776aec5c OK. 2023-10-02 12:24:32,841 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-10-02 12:24:32,841 main DEBUG OutputStream closed 2023-10-02 12:24:32,841 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-10-02 12:24:32,842 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-10-02 12:24:32,842 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@3daa422a OK 2023-10-02 12:24:32,895 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-10-02 12:24:32,899 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-10-02 12:24:32,902 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-10-02 12:24:32,905 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-10-02 12:24:32,906 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-10-02 12:24:32,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-10-02 12:24:32,907 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-10-02 12:24:32,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-10-02 12:24:32,908 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-10-02 12:24:32,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-10-02 12:24:32,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-10-02 12:24:32,909 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-10-02 12:24:32,910 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-10-02 12:24:32,910 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-10-02 12:24:32,910 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-10-02 12:24:32,911 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-10-02 12:24:32,911 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-10-02 12:24:32,916 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-10-02 12:24:32,920 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-10-02 12:24:32,921 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@15bbf42f) with optional ClassLoader: null 2023-10-02 12:24:32,921 main DEBUG Shutdown hook enabled. Registering a new one. 2023-10-02 12:24:32,922 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@15bbf42f] started OK. 2023-10-02T12:24:32,958 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures timeout: 13 mins 2023-10-02 12:24:32,963 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-10-02 12:24:32,963 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-10-02T12:24:33,442 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0 2023-10-02T12:24:33,448 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-10-02T12:24:33,451 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:24:33,453 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-10-02T12:24:33,457 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/cluster_7356682d-271e-efa6-01f2-4b2f0febdaf5, deleteOnExit=true 2023-10-02T12:24:33,458 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-10-02T12:24:33,458 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/test.cache.data in system properties and HBase conf 2023-10-02T12:24:33,459 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/hadoop.tmp.dir in system properties and HBase conf 2023-10-02T12:24:33,460 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/hadoop.log.dir in system properties and HBase conf 2023-10-02T12:24:33,460 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/mapreduce.cluster.local.dir in system properties and HBase conf 2023-10-02T12:24:33,461 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-10-02T12:24:33,461 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-10-02T12:24:33,563 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-10-02T12:24:34,140 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-10-02T12:24:34,145 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:24:34,145 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:24:34,146 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-10-02T12:24:34,146 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:24:34,147 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-10-02T12:24:34,147 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-10-02T12:24:34,148 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:24:34,148 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:24:34,149 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-10-02T12:24:34,149 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/nfs.dump.dir in system properties and HBase conf 2023-10-02T12:24:34,150 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/java.io.tmpdir in system properties and HBase conf 2023-10-02T12:24:34,150 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:24:34,151 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-10-02T12:24:34,151 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-10-02T12:24:34,672 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-10-02T12:24:34,676 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-10-02T12:24:34,975 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-10-02T12:24:35,113 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@24a32549 via org.mortbay.log.Slf4jLog 2023-10-02T12:24:35,129 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:24:35,170 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:24:35,207 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/java.io.tmpdir/Jetty_localhost_33841_hdfs____.bqfsox/webapp 2023-10-02T12:24:35,347 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33841 2023-10-02T12:24:35,375 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-10-02T12:24:35,375 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-10-02T12:24:36,030 WARN [Listener at localhost/36497 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:24:36,160 WARN [Listener at localhost/36497 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-10-02T12:24:36,186 WARN [Listener at localhost/36497 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:24:36,194 INFO [Listener at localhost/36497 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:24:36,202 INFO [Listener at localhost/36497 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/java.io.tmpdir/Jetty_localhost_34895_datanode____.e3kv9i/webapp 2023-10-02T12:24:36,327 INFO [Listener at localhost/36497 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:34895 2023-10-02T12:24:36,754 WARN [Listener at localhost/43179 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:24:37,138 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-10-02T12:24:37,498 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x6ccf3fc9f9ef8e28: Processing first storage report for DS-d3b004e5-e24b-4408-a9bf-421ca1308ded from datanode 2484e56c-fb12-4c55-8990-9f71d5262c43 2023-10-02T12:24:37,500 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x6ccf3fc9f9ef8e28: from storage DS-d3b004e5-e24b-4408-a9bf-421ca1308ded node DatanodeRegistration(127.0.0.1:38869, datanodeUuid=2484e56c-fb12-4c55-8990-9f71d5262c43, infoPort=32993, infoSecurePort=0, ipcPort=43179, storageInfo=lv=-57;cid=testClusterID;nsid=1699211796;c=1696249474750), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2023-10-02T12:24:37,500 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x6ccf3fc9f9ef8e28: Processing first storage report for DS-529448b9-8f1a-4660-9086-bd1ce67b9059 from datanode 2484e56c-fb12-4c55-8990-9f71d5262c43 2023-10-02T12:24:37,500 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x6ccf3fc9f9ef8e28: from storage DS-529448b9-8f1a-4660-9086-bd1ce67b9059 node DatanodeRegistration(127.0.0.1:38869, datanodeUuid=2484e56c-fb12-4c55-8990-9f71d5262c43, infoPort=32993, infoSecurePort=0, ipcPort=43179, storageInfo=lv=-57;cid=testClusterID;nsid=1699211796;c=1696249474750), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-10-02T12:24:37,534 DEBUG [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0 2023-10-02T12:24:37,645 INFO [Listener at localhost/43179 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/cluster_7356682d-271e-efa6-01f2-4b2f0febdaf5/zookeeper_0, clientPort=58172, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/cluster_7356682d-271e-efa6-01f2-4b2f0febdaf5/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/cluster_7356682d-271e-efa6-01f2-4b2f0febdaf5/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-10-02T12:24:37,672 INFO [Listener at localhost/43179 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=58172 2023-10-02T12:24:37,683 INFO [Listener at localhost/43179 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:24:37,686 INFO [Listener at localhost/43179 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:24:38,453 INFO [Listener at localhost/43179 {}] util.FSUtils(462): Created version file at hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a with version=8 2023-10-02T12:24:38,453 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/hbase-staging 2023-10-02T12:24:38,474 DEBUG [Listener at localhost/43179 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-10-02T12:24:38,475 DEBUG [Listener at localhost/43179 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-10-02T12:24:38,475 DEBUG [Listener at localhost/43179 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-10-02T12:24:38,475 DEBUG [Listener at localhost/43179 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-10-02T12:24:38,736 DEBUG [Listener at localhost/43179 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-10-02T12:24:39,642 INFO [Listener at localhost/43179 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=3 2023-10-02T12:24:39,702 INFO [Listener at localhost/43179 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:24:39,703 INFO [Listener at localhost/43179 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-10-02T12:24:39,703 INFO [Listener at localhost/43179 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-10-02T12:24:39,703 INFO [Listener at localhost/43179 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:24:39,704 INFO [Listener at localhost/43179 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-10-02T12:24:39,900 INFO [Listener at localhost/43179 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-10-02T12:24:40,058 INFO [Listener at localhost/43179 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-10-02T12:24:40,080 DEBUG [Listener at localhost/43179 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-10-02T12:24:40,087 INFO [Listener at localhost/43179 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-10-02T12:24:40,163 DEBUG [Listener at localhost/43179 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 12637 (auto-detected) 2023-10-02T12:24:40,165 DEBUG [Listener at localhost/43179 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-10-02T12:24:40,223 INFO [Listener at localhost/43179 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:43611 2023-10-02T12:24:40,242 INFO [Listener at localhost/43179 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:24:40,244 INFO [Listener at localhost/43179 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:24:40,269 INFO [Listener at localhost/43179 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:43611 connecting to ZooKeeper ensemble=127.0.0.1:58172 2023-10-02T12:24:40,325 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:436110x0, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-10-02T12:24:40,329 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:43611-0x10173225a500000 connected 2023-10-02T12:24:40,367 DEBUG [Listener at localhost/43179 {}] zookeeper.ZKUtil(113): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-10-02T12:24:40,372 DEBUG [Listener at localhost/43179 {}] zookeeper.ZKUtil(113): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-10-02T12:24:40,377 DEBUG [Listener at localhost/43179 {}] zookeeper.ZKUtil(113): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-10-02T12:24:40,389 DEBUG [Listener at localhost/43179 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43611 2023-10-02T12:24:40,390 DEBUG [Listener at localhost/43179 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43611 2023-10-02T12:24:40,391 DEBUG [Listener at localhost/43179 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43611 2023-10-02T12:24:40,392 DEBUG [Listener at localhost/43179 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43611 2023-10-02T12:24:40,393 DEBUG [Listener at localhost/43179 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43611 2023-10-02T12:24:40,462 INFO [Listener at localhost/43179 {}] log.Log(170): Logging initialized @8569ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-10-02T12:24:40,612 INFO [Listener at localhost/43179 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-10-02T12:24:40,613 INFO [Listener at localhost/43179 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-10-02T12:24:40,613 INFO [Listener at localhost/43179 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-10-02T12:24:40,616 INFO [Listener at localhost/43179 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-10-02T12:24:40,616 INFO [Listener at localhost/43179 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-10-02T12:24:40,616 INFO [Listener at localhost/43179 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-10-02T12:24:40,620 INFO [Listener at localhost/43179 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-10-02T12:24:40,786 INFO [Listener at localhost/43179 {}] http.HttpServer(1219): Jetty bound to port 44299 2023-10-02T12:24:40,789 INFO [Listener at localhost/43179 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:24:40,837 INFO [Listener at localhost/43179 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:24:40,842 INFO [Listener at localhost/43179 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@6a56522d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/hadoop.log.dir/,AVAILABLE} 2023-10-02T12:24:40,843 INFO [Listener at localhost/43179 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:24:40,843 INFO [Listener at localhost/43179 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5f5c5c02{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-10-02T12:24:40,946 INFO [Listener at localhost/43179 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-10-02T12:24:40,969 INFO [Listener at localhost/43179 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-10-02T12:24:40,970 INFO [Listener at localhost/43179 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-10-02T12:24:40,972 INFO [Listener at localhost/43179 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-10-02T12:24:40,980 INFO [Listener at localhost/43179 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:24:42,215 INFO [Listener at localhost/43179 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@3e051dc7{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-10-02T12:24:42,227 INFO [Listener at localhost/43179 {}] server.AbstractConnector(333): Started ServerConnector@33782983{HTTP/1.1, (http/1.1)}{0.0.0.0:44299} 2023-10-02T12:24:42,227 INFO [Listener at localhost/43179 {}] server.Server(415): Started @10334ms 2023-10-02T12:24:42,231 INFO [Listener at localhost/43179 {}] master.HMaster(485): hbase.rootdir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a, hbase.cluster.distributed=false 2023-10-02T12:24:42,295 INFO [Listener at localhost/43179 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=3 2023-10-02T12:24:42,296 INFO [Listener at localhost/43179 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:24:42,296 INFO [Listener at localhost/43179 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-10-02T12:24:42,296 INFO [Listener at localhost/43179 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-10-02T12:24:42,296 INFO [Listener at localhost/43179 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:24:42,296 INFO [Listener at localhost/43179 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-10-02T12:24:42,305 INFO [Listener at localhost/43179 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-10-02T12:24:42,307 INFO [Listener at localhost/43179 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-10-02T12:24:42,308 INFO [Listener at localhost/43179 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:35033 2023-10-02T12:24:42,311 INFO [Listener at localhost/43179 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-10-02T12:24:42,319 DEBUG [Listener at localhost/43179 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-10-02T12:24:42,320 INFO [Listener at localhost/43179 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:24:42,323 INFO [Listener at localhost/43179 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:24:42,326 INFO [Listener at localhost/43179 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:35033 connecting to ZooKeeper ensemble=127.0.0.1:58172 2023-10-02T12:24:42,329 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): regionserver:350330x0, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-10-02T12:24:42,330 DEBUG [Listener at localhost/43179 {}] zookeeper.ZKUtil(113): regionserver:350330x0, quorum=127.0.0.1:58172, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-10-02T12:24:42,331 DEBUG [Listener at localhost/43179 {}] zookeeper.ZKUtil(113): regionserver:350330x0, quorum=127.0.0.1:58172, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-10-02T12:24:42,332 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:35033-0x10173225a500001 connected 2023-10-02T12:24:42,332 DEBUG [Listener at localhost/43179 {}] zookeeper.ZKUtil(113): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-10-02T12:24:42,333 DEBUG [Listener at localhost/43179 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35033 2023-10-02T12:24:42,334 DEBUG [Listener at localhost/43179 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35033 2023-10-02T12:24:42,334 DEBUG [Listener at localhost/43179 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35033 2023-10-02T12:24:42,334 DEBUG [Listener at localhost/43179 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35033 2023-10-02T12:24:42,335 DEBUG [Listener at localhost/43179 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35033 2023-10-02T12:24:42,339 INFO [Listener at localhost/43179 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-10-02T12:24:42,339 INFO [Listener at localhost/43179 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-10-02T12:24:42,339 INFO [Listener at localhost/43179 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-10-02T12:24:42,340 INFO [Listener at localhost/43179 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-10-02T12:24:42,340 INFO [Listener at localhost/43179 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-10-02T12:24:42,340 INFO [Listener at localhost/43179 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-10-02T12:24:42,341 INFO [Listener at localhost/43179 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-10-02T12:24:42,343 INFO [Listener at localhost/43179 {}] http.HttpServer(1219): Jetty bound to port 33875 2023-10-02T12:24:42,344 INFO [Listener at localhost/43179 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:24:42,345 INFO [Listener at localhost/43179 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:24:42,346 INFO [Listener at localhost/43179 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@4abd68ea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/hadoop.log.dir/,AVAILABLE} 2023-10-02T12:24:42,346 INFO [Listener at localhost/43179 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:24:42,346 INFO [Listener at localhost/43179 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@61caa118{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-10-02T12:24:42,359 INFO [Listener at localhost/43179 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-10-02T12:24:42,360 INFO [Listener at localhost/43179 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-10-02T12:24:42,361 INFO [Listener at localhost/43179 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-10-02T12:24:42,361 INFO [Listener at localhost/43179 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-10-02T12:24:42,362 INFO [Listener at localhost/43179 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:24:42,367 INFO [Listener at localhost/43179 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@584c4016{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-10-02T12:24:42,368 INFO [Listener at localhost/43179 {}] server.AbstractConnector(333): Started ServerConnector@319cc17a{HTTP/1.1, (http/1.1)}{0.0.0.0:33875} 2023-10-02T12:24:42,368 INFO [Listener at localhost/43179 {}] server.Server(415): Started @10475ms 2023-10-02T12:24:42,401 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:24:42,408 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@58bcb309{HTTP/1.1, (http/1.1)}{0.0.0.0:34987} 2023-10-02T12:24:42,408 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @10515ms 2023-10-02T12:24:42,408 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2388): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,43611,1696249478718 2023-10-02T12:24:42,419 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:24:42,420 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:24:42,422 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,43611,1696249478718 2023-10-02T12:24:42,425 DEBUG [M:0;jenkins-hbase3:43611 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:43611 2023-10-02T12:24:42,444 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-10-02T12:24:42,444 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-10-02T12:24:42,445 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:24:42,445 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:24:42,448 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Set watcher on existing znode=/1/master 2023-10-02T12:24:42,449 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Set watcher on existing znode=/1/master 2023-10-02T12:24:42,449 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,43611,1696249478718 from backup master directory 2023-10-02T12:24:42,454 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:24:42,454 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,43611,1696249478718 2023-10-02T12:24:42,455 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:24:42,456 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-10-02T12:24:42,456 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,43611,1696249478718 2023-10-02T12:24:42,460 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-10-02T12:24:42,463 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-10-02T12:24:42,590 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/hbase.id with ID: a892db5d-3f1f-4425-bfaf-3139ae30aa67 2023-10-02T12:24:42,651 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:24:42,669 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:24:42,669 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:24:42,730 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:24:42,733 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-10-02T12:24:42,758 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-10-02T12:24:42,758 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-10-02T12:24:42,760 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:24:42,773 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:24:42,775 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:24:42,899 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/data/master/store 2023-10-02T12:24:43,325 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-10-02T12:24:43,326 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:24:43,326 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-10-02T12:24:43,326 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:24:43,326 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:24:43,327 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-10-02T12:24:43,327 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:24:43,327 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:24:43,327 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:24:43,329 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/data/master/store/.initializing 2023-10-02T12:24:43,330 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/WALs/jenkins-hbase3.apache.org,43611,1696249478718 2023-10-02T12:24:43,340 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-10-02T12:24:43,355 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C43611%2C1696249478718, suffix=, logDir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/WALs/jenkins-hbase3.apache.org,43611,1696249478718, archiveDir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/oldWALs, maxLogs=10 2023-10-02T12:24:43,384 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/WALs/jenkins-hbase3.apache.org,43611,1696249478718/jenkins-hbase3.apache.org%2C43611%2C1696249478718.1696249483362, exclude list is [], retry=0 2023-10-02T12:24:43,411 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38869,DS-d3b004e5-e24b-4408-a9bf-421ca1308ded,DISK] 2023-10-02T12:24:43,419 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:24:43,497 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/WALs/jenkins-hbase3.apache.org,43611,1696249478718/jenkins-hbase3.apache.org%2C43611%2C1696249478718.1696249483362 2023-10-02T12:24:43,498 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38869,DS-d3b004e5-e24b-4408-a9bf-421ca1308ded,DISK]] 2023-10-02T12:24:43,498 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:24:43,499 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:24:43,503 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:24:43,504 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:24:43,572 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:24:43,608 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-10-02T12:24:43,613 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:43,617 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:24:43,618 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:24:43,622 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-10-02T12:24:43,622 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:43,623 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:24:43,623 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:24:43,627 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-10-02T12:24:43,627 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:43,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:24:43,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:24:43,631 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-10-02T12:24:43,631 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:43,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:24:43,638 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:24:43,639 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:24:43,660 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-10-02T12:24:43,664 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:24:43,669 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:24:43,670 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11193361120, jitterRate=0.04246298968791962}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-10-02T12:24:43,679 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:24:43,680 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-10-02T12:24:43,718 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c0b0463, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:24:43,776 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(860): No meta location available on zookeeper, skip migrating... 2023-10-02T12:24:43,798 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-10-02T12:24:43,798 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-10-02T12:24:43,802 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-10-02T12:24:43,803 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-10-02T12:24:43,812 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 8 msec 2023-10-02T12:24:43,812 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-10-02T12:24:43,863 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-10-02T12:24:43,880 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-10-02T12:24:43,882 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-10-02T12:24:43,886 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-10-02T12:24:43,887 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-10-02T12:24:43,889 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-10-02T12:24:43,891 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-10-02T12:24:43,898 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-10-02T12:24:43,899 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-10-02T12:24:43,900 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-10-02T12:24:43,904 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-10-02T12:24:43,920 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-10-02T12:24:43,921 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-10-02T12:24:43,927 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-10-02T12:24:43,927 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-10-02T12:24:43,927 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:24:43,927 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:24:43,928 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(804): Active/primary master=jenkins-hbase3.apache.org,43611,1696249478718, sessionid=0x10173225a500000, setting cluster-up flag (Was=false) 2023-10-02T12:24:43,941 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-10-02T12:24:43,951 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:24:43,951 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:24:43,956 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-10-02T12:24:43,958 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,43611,1696249478718 2023-10-02T12:24:43,967 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:24:43,968 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:24:43,976 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-10-02T12:24:43,977 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,43611,1696249478718 2023-10-02T12:24:43,982 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:24:43,982 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:24:43,989 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-10-02T12:24:43,990 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,43611,1696249478718 2023-10-02T12:24:44,059 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver loaded, priority=536870911. 2023-10-02T12:24:44,110 DEBUG [RS:0;jenkins-hbase3:35033 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:35033 2023-10-02T12:24:44,112 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(997): ClusterId : a892db5d-3f1f-4425-bfaf-3139ae30aa67 2023-10-02T12:24:44,114 INFO [RS:0;jenkins-hbase3:35033 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-10-02T12:24:44,117 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-10-02T12:24:44,124 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-10-02T12:24:44,124 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-10-02T12:24:44,128 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-10-02T12:24:44,128 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-10-02T12:24:44,132 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-10-02T12:24:44,134 DEBUG [RS:0;jenkins-hbase3:35033 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b8e1764, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:24:44,136 DEBUG [RS:0;jenkins-hbase3:35033 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2382e507, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-10-02T12:24:44,145 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-10-02T12:24:44,145 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-10-02T12:24:44,145 DEBUG [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-10-02T12:24:44,150 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,43611,1696249478718 with isa=jenkins-hbase3.apache.org/172.31.12.81:35033, startcode=1696249482294 2023-10-02T12:24:44,153 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-10-02T12:24:44,164 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-10-02T12:24:44,169 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-10-02T12:24:44,178 DEBUG [RS:0;jenkins-hbase3:35033 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-10-02T12:24:44,178 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,43611,1696249478718 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-10-02T12:24:44,183 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:24:44,183 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:24:44,183 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:24:44,183 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:24:44,183 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-10-02T12:24:44,183 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:24:44,183 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-10-02T12:24:44,183 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:24:44,185 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1696249514185 2023-10-02T12:24:44,187 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-10-02T12:24:44,194 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-10-02T12:24:44,198 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-10-02T12:24:44,199 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-10-02T12:24:44,204 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:44,204 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-10-02T12:24:44,205 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-10-02T12:24:44,206 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-10-02T12:24:44,206 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-10-02T12:24:44,207 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-10-02T12:24:44,207 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-10-02T12:24:44,210 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-10-02T12:24:44,212 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-10-02T12:24:44,216 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-10-02T12:24:44,216 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-10-02T12:24:44,232 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0f402e49 to 127.0.0.1:58172 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:24:44,263 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f9b10f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:24:44,264 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-10-02T12:24:44,268 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-10-02T12:24:44,268 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-10-02T12:24:44,273 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696249484270,5,FailOnTimeoutGroup] 2023-10-02T12:24:44,275 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57901, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-10-02T12:24:44,276 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696249484274,5,FailOnTimeoutGroup] 2023-10-02T12:24:44,276 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-10-02T12:24:44,276 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1635): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-10-02T12:24:44,278 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-10-02T12:24:44,279 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-10-02T12:24:44,290 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43611 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:24:44,316 DEBUG [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a 2023-10-02T12:24:44,317 DEBUG [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:36497 2023-10-02T12:24:44,317 DEBUG [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=44299 2023-10-02T12:24:44,323 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-10-02T12:24:44,324 DEBUG [RS:0;jenkins-hbase3:35033 {}] zookeeper.ZKUtil(111): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:24:44,324 WARN [RS:0;jenkins-hbase3:35033 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-10-02T12:24:44,324 INFO [RS:0;jenkins-hbase3:35033 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:24:44,324 DEBUG [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:24:44,328 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,35033,1696249482294] 2023-10-02T12:24:44,343 DEBUG [RS:0;jenkins-hbase3:35033 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-10-02T12:24:44,358 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-10-02T12:24:44,372 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-10-02T12:24:44,377 INFO [RS:0;jenkins-hbase3:35033 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-10-02T12:24:44,377 INFO [RS:0;jenkins-hbase3:35033 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:24:44,378 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-10-02T12:24:44,388 INFO [RS:0;jenkins-hbase3:35033 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-10-02T12:24:44,388 DEBUG [RS:0;jenkins-hbase3:35033 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:24:44,388 DEBUG [RS:0;jenkins-hbase3:35033 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:24:44,388 DEBUG [RS:0;jenkins-hbase3:35033 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:24:44,388 DEBUG [RS:0;jenkins-hbase3:35033 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:24:44,389 DEBUG [RS:0;jenkins-hbase3:35033 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:24:44,389 DEBUG [RS:0;jenkins-hbase3:35033 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-10-02T12:24:44,389 DEBUG [RS:0;jenkins-hbase3:35033 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:24:44,389 DEBUG [RS:0;jenkins-hbase3:35033 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:24:44,389 DEBUG [RS:0;jenkins-hbase3:35033 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:24:44,389 DEBUG [RS:0;jenkins-hbase3:35033 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:24:44,389 DEBUG [RS:0;jenkins-hbase3:35033 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:24:44,389 DEBUG [RS:0;jenkins-hbase3:35033 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-10-02T12:24:44,390 DEBUG [RS:0;jenkins-hbase3:35033 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-10-02T12:24:44,391 INFO [RS:0;jenkins-hbase3:35033 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-10-02T12:24:44,391 INFO [RS:0;jenkins-hbase3:35033 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-10-02T12:24:44,391 INFO [RS:0;jenkins-hbase3:35033 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-10-02T12:24:44,391 INFO [RS:0;jenkins-hbase3:35033 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-10-02T12:24:44,391 INFO [RS:0;jenkins-hbase3:35033 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,35033,1696249482294-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-10-02T12:24:44,409 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-10-02T12:24:44,412 INFO [RS:0;jenkins-hbase3:35033 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,35033,1696249482294-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:24:44,427 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,35033,1696249482294 started 2023-10-02T12:24:44,428 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,35033,1696249482294, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:35033, sessionid=0x10173225a500001 2023-10-02T12:24:44,428 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-10-02T12:24:44,428 DEBUG [RS:0;jenkins-hbase3:35033 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:24:44,428 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,35033,1696249482294' 2023-10-02T12:24:44,428 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-10-02T12:24:44,429 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-10-02T12:24:44,430 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-10-02T12:24:44,430 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-10-02T12:24:44,430 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,35033,1696249482294' 2023-10-02T12:24:44,430 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:24:44,431 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:24:44,431 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-10-02T12:24:44,431 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-10-02T12:24:44,431 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-10-02T12:24:44,431 DEBUG [RS:0;jenkins-hbase3:35033 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:24:44,431 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,35033,1696249482294' 2023-10-02T12:24:44,431 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-10-02T12:24:44,432 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-10-02T12:24:44,432 DEBUG [RS:0;jenkins-hbase3:35033 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-10-02T12:24:44,432 INFO [RS:0;jenkins-hbase3:35033 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-10-02T12:24:44,433 INFO [RS:0;jenkins-hbase3:35033 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-10-02T12:24:44,543 INFO [RS:0;jenkins-hbase3:35033 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-10-02T12:24:44,546 INFO [RS:0;jenkins-hbase3:35033 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C35033%2C1696249482294, suffix=, logDir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294, archiveDir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/oldWALs, maxLogs=32 2023-10-02T12:24:44,558 DEBUG [RS:0;jenkins-hbase3:35033 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.1696249484548, exclude list is [], retry=0 2023-10-02T12:24:44,565 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38869,DS-d3b004e5-e24b-4408-a9bf-421ca1308ded,DISK] 2023-10-02T12:24:44,570 INFO [RS:0;jenkins-hbase3:35033 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.1696249484548 2023-10-02T12:24:44,571 DEBUG [RS:0;jenkins-hbase3:35033 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38869,DS-d3b004e5-e24b-4408-a9bf-421ca1308ded,DISK]] 2023-10-02T12:24:44,662 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-10-02T12:24:44,662 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a 2023-10-02T12:24:45,076 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:24:45,079 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-10-02T12:24:45,081 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-10-02T12:24:45,082 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:45,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:24:45,083 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-10-02T12:24:45,085 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-10-02T12:24:45,085 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:45,086 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:24:45,086 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-10-02T12:24:45,088 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-10-02T12:24:45,088 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:45,089 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:24:45,090 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/meta/1588230740 2023-10-02T12:24:45,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/meta/1588230740 2023-10-02T12:24:45,094 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-10-02T12:24:45,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-10-02T12:24:45,100 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:24:45,100 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10571061120, jitterRate=-0.015493214130401611}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-10-02T12:24:45,102 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-10-02T12:24:45,102 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-10-02T12:24:45,102 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-10-02T12:24:45,102 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-10-02T12:24:45,102 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-10-02T12:24:45,102 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-10-02T12:24:45,104 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-10-02T12:24:45,104 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-10-02T12:24:45,107 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-10-02T12:24:45,107 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-10-02T12:24:45,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-10-02T12:24:45,131 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-10-02T12:24:45,136 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-10-02T12:24:45,288 DEBUG [jenkins-hbase3:43611 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-10-02T12:24:45,296 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:24:45,303 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,35033,1696249482294, state=OPENING 2023-10-02T12:24:45,311 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-10-02T12:24:45,319 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:24:45,319 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:24:45,320 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:24:45,320 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:24:45,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,35033,1696249482294}] 2023-10-02T12:24:45,516 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:24:45,518 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-10-02T12:24:45,522 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42044, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-10-02T12:24:45,535 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-10-02T12:24:45,536 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:24:45,537 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-10-02T12:24:45,539 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta, suffix=.meta, logDir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294, archiveDir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/oldWALs, maxLogs=32 2023-10-02T12:24:45,555 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta.1696249485541.meta, exclude list is [], retry=0 2023-10-02T12:24:45,559 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38869,DS-d3b004e5-e24b-4408-a9bf-421ca1308ded,DISK] 2023-10-02T12:24:45,564 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta.1696249485541.meta 2023-10-02T12:24:45,564 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38869,DS-d3b004e5-e24b-4408-a9bf-421ca1308ded,DISK]] 2023-10-02T12:24:45,565 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:24:45,568 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:24:45,569 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-10-02T12:24:45,586 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-10-02T12:24:45,586 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-10-02T12:24:45,591 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-10-02T12:24:45,591 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:24:45,592 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-10-02T12:24:45,592 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-10-02T12:24:45,594 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-10-02T12:24:45,596 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-10-02T12:24:45,596 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:45,597 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:24:45,597 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-10-02T12:24:45,598 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-10-02T12:24:45,598 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:45,599 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:24:45,599 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-10-02T12:24:45,600 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-10-02T12:24:45,600 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:45,601 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:24:45,603 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/meta/1588230740 2023-10-02T12:24:45,605 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/meta/1588230740 2023-10-02T12:24:45,608 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-10-02T12:24:45,611 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-10-02T12:24:45,612 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9812032320, jitterRate=-0.08618327975273132}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-10-02T12:24:45,613 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-10-02T12:24:45,626 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1696249485507 2023-10-02T12:24:45,646 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-10-02T12:24:45,647 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-10-02T12:24:45,648 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:24:45,651 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,35033,1696249482294, state=OPEN 2023-10-02T12:24:45,654 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-10-02T12:24:45,654 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-10-02T12:24:45,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:24:45,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:24:45,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-10-02T12:24:45,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,35033,1696249482294 in 331 msec 2023-10-02T12:24:45,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-10-02T12:24:45,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 544 msec 2023-10-02T12:24:45,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.7990 sec 2023-10-02T12:24:45,868 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1063): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1696249485868, completionTime=-1 2023-10-02T12:24:45,868 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-10-02T12:24:45,869 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-10-02T12:24:45,937 DEBUG [hconnection-0x4bad9d8a-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:24:45,942 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42048, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:24:45,959 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-10-02T12:24:45,959 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1696249545959 2023-10-02T12:24:45,959 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1696249605959 2023-10-02T12:24:45,959 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 90 msec 2023-10-02T12:24:45,987 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43611,1696249478718-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:24:45,987 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43611,1696249478718-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:24:45,988 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43611,1696249478718-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:24:45,990 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:43611, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:24:45,991 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-10-02T12:24:46,003 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-10-02T12:24:46,020 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-10-02T12:24:46,022 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2368): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-10-02T12:24:46,036 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-10-02T12:24:46,041 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:24:46,044 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:46,047 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:24:46,083 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 8e88a763331d94795997dc16a6a7ee9b, NAME => 'hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a 2023-10-02T12:24:46,098 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:24:46,098 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing 8e88a763331d94795997dc16a6a7ee9b, disabling compactions & flushes 2023-10-02T12:24:46,098 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b. 2023-10-02T12:24:46,098 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b. 2023-10-02T12:24:46,099 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b. after waiting 0 ms 2023-10-02T12:24:46,099 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b. 2023-10-02T12:24:46,099 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b. 2023-10-02T12:24:46,099 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for 8e88a763331d94795997dc16a6a7ee9b: 2023-10-02T12:24:46,102 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:24:46,123 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1696249486104"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249486104"}]},"ts":"1696249486104"} 2023-10-02T12:24:46,198 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:24:46,201 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:24:46,209 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249486201"}]},"ts":"1696249486201"} 2023-10-02T12:24:46,215 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-10-02T12:24:46,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=8e88a763331d94795997dc16a6a7ee9b, ASSIGN}] 2023-10-02T12:24:46,225 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=8e88a763331d94795997dc16a6a7ee9b, ASSIGN 2023-10-02T12:24:46,227 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=8e88a763331d94795997dc16a6a7ee9b, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,35033,1696249482294; forceNewPlan=false, retain=false 2023-10-02T12:24:46,378 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=8e88a763331d94795997dc16a6a7ee9b, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:24:46,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 8e88a763331d94795997dc16a6a7ee9b, server=jenkins-hbase3.apache.org,35033,1696249482294}] 2023-10-02T12:24:46,546 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b. 2023-10-02T12:24:46,547 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => 8e88a763331d94795997dc16a6a7ee9b, NAME => 'hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:24:46,548 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:24:46,548 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 8e88a763331d94795997dc16a6a7ee9b 2023-10-02T12:24:46,549 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:24:46,549 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for 8e88a763331d94795997dc16a6a7ee9b 2023-10-02T12:24:46,549 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for 8e88a763331d94795997dc16a6a7ee9b 2023-10-02T12:24:46,551 INFO [StoreOpener-8e88a763331d94795997dc16a6a7ee9b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 8e88a763331d94795997dc16a6a7ee9b 2023-10-02T12:24:46,553 INFO [StoreOpener-8e88a763331d94795997dc16a6a7ee9b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e88a763331d94795997dc16a6a7ee9b columnFamilyName info 2023-10-02T12:24:46,553 DEBUG [StoreOpener-8e88a763331d94795997dc16a6a7ee9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:46,554 INFO [StoreOpener-8e88a763331d94795997dc16a6a7ee9b-1 {}] regionserver.HStore(324): Store=8e88a763331d94795997dc16a6a7ee9b/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:24:46,555 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/namespace/8e88a763331d94795997dc16a6a7ee9b 2023-10-02T12:24:46,556 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/namespace/8e88a763331d94795997dc16a6a7ee9b 2023-10-02T12:24:46,559 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for 8e88a763331d94795997dc16a6a7ee9b 2023-10-02T12:24:46,563 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/namespace/8e88a763331d94795997dc16a6a7ee9b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:24:46,564 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened 8e88a763331d94795997dc16a6a7ee9b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10958280480, jitterRate=0.020569398999214172}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:24:46,565 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for 8e88a763331d94795997dc16a6a7ee9b: 2023-10-02T12:24:46,566 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b., pid=6, masterSystemTime=1696249486540 2023-10-02T12:24:46,570 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b. 2023-10-02T12:24:46,570 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b. 2023-10-02T12:24:46,571 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=8e88a763331d94795997dc16a6a7ee9b, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:24:46,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-10-02T12:24:46,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 8e88a763331d94795997dc16a6a7ee9b, server=jenkins-hbase3.apache.org,35033,1696249482294 in 189 msec 2023-10-02T12:24:46,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-10-02T12:24:46,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=8e88a763331d94795997dc16a6a7ee9b, ASSIGN in 356 msec 2023-10-02T12:24:46,583 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:24:46,583 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249486583"}]},"ts":"1696249486583"} 2023-10-02T12:24:46,586 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-10-02T12:24:46,589 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:24:46,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 565 msec 2023-10-02T12:24:46,644 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-10-02T12:24:46,646 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-10-02T12:24:46,646 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:24:46,646 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:24:46,704 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-10-02T12:24:46,723 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:24:46,729 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 37 msec 2023-10-02T12:24:46,741 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-10-02T12:24:46,754 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:24:46,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 17 msec 2023-10-02T12:24:46,769 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-10-02T12:24:46,771 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-10-02T12:24:46,771 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1193): Master has completed initialization 4.315sec 2023-10-02T12:24:46,773 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-10-02T12:24:46,775 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-10-02T12:24:46,776 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-10-02T12:24:46,777 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-10-02T12:24:46,777 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-10-02T12:24:46,780 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43611,1696249478718-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-10-02T12:24:46,780 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,43611,1696249478718-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-10-02T12:24:46,796 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1296): Balancer post startup initialization complete, took 0 seconds 2023-10-02T12:24:46,798 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-10-02T12:24:46,871 DEBUG [Listener at localhost/43179 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x059428ad to 127.0.0.1:58172 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:24:46,877 DEBUG [Listener at localhost/43179 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64e41985, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:24:46,884 DEBUG [Listener at localhost/43179 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-10-02T12:24:46,884 DEBUG [Listener at localhost/43179 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-10-02T12:24:46,903 DEBUG [hconnection-0xed8f3b0-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:24:46,915 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42052, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:24:46,926 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,43611,1696249478718 2023-10-02T12:24:46,926 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-10-02T12:24:46,926 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/test.cache.data in system properties and HBase conf 2023-10-02T12:24:46,926 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/hadoop.tmp.dir in system properties and HBase conf 2023-10-02T12:24:46,926 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/hadoop.log.dir in system properties and HBase conf 2023-10-02T12:24:46,926 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/mapreduce.cluster.local.dir in system properties and HBase conf 2023-10-02T12:24:46,926 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-10-02T12:24:46,927 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-10-02T12:24:46,927 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:24:46,927 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:24:46,927 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-10-02T12:24:46,927 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:24:46,927 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-10-02T12:24:46,927 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-10-02T12:24:46,927 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:24:46,927 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:24:46,927 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-10-02T12:24:46,927 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/nfs.dump.dir in system properties and HBase conf 2023-10-02T12:24:46,927 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/java.io.tmpdir in system properties and HBase conf 2023-10-02T12:24:46,927 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:24:46,928 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-10-02T12:24:46,928 INFO [Listener at localhost/43179 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-10-02T12:24:49,068 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:24:49,489 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:24:49,495 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_36499_jobhistory____qigdqz/webapp 2023-10-02T12:24:49,680 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-10-02T12:24:49,680 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-10-02T12:24:49,680 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:24:49,684 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:24:49,780 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:24:49,925 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:24:50,053 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-10-02T12:24:50,053 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-10-02T12:24:50,090 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-10-02T12:24:50,090 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-10-02T12:24:50,092 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: Master,sub=Coprocessor.Master.CP_org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver 2023-10-02T12:24:50,093 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering Master,sub=Coprocessor.Master.CP_org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver Metrics about HBase MasterObservers 2023-10-02T12:24:50,093 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:24:50,093 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-10-02T12:24:50,093 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-02T12:24:50,093 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-10-02T12:24:50,094 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-10-02T12:24:50,094 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-10-02T12:24:50,300 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-10-02T12:24:50,343 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:36499 2023-10-02T12:24:50,351 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-10-02T12:24:50,353 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-10-02T12:24:50,562 WARN [Listener at jenkins-hbase3.apache.org/44831 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:24:50,734 WARN [Listener at jenkins-hbase3.apache.org/44831 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:24:50,735 INFO [Listener at jenkins-hbase3.apache.org/44831 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:24:50,819 INFO [Listener at jenkins-hbase3.apache.org/44831 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_45657_cluster____kk293g/webapp 2023-10-02T12:24:50,912 INFO [Listener at jenkins-hbase3.apache.org/44831 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:24:50,913 INFO [Listener at jenkins-hbase3.apache.org/44831 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-10-02T12:24:50,913 INFO [Listener at jenkins-hbase3.apache.org/44831 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:24:50,913 INFO [Listener at jenkins-hbase3.apache.org/44831 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:24:50,921 INFO [Listener at jenkins-hbase3.apache.org/44831 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:24:51,157 INFO [Listener at jenkins-hbase3.apache.org/44831 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:24:51,208 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:24:51,316 INFO [Listener at jenkins-hbase3.apache.org/44831 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:24:51,329 INFO [Listener at jenkins-hbase3.apache.org/44831 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:45657 2023-10-02T12:24:51,863 WARN [Listener at jenkins-hbase3.apache.org/45745 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:24:51,892 WARN [Listener at jenkins-hbase3.apache.org/45745 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:24:51,893 INFO [Listener at jenkins-hbase3.apache.org/45745 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:24:51,901 INFO [Listener at jenkins-hbase3.apache.org/45745 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_43799_node____.rdrn6x/webapp 2023-10-02T12:24:51,968 INFO [Listener at jenkins-hbase3.apache.org/45745 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-10-02T12:24:51,968 INFO [Listener at jenkins-hbase3.apache.org/45745 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:24:51,968 INFO [Listener at jenkins-hbase3.apache.org/45745 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:24:51,968 INFO [Listener at jenkins-hbase3.apache.org/45745 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:24:51,977 INFO [Listener at jenkins-hbase3.apache.org/45745 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:24:52,003 INFO [Listener at jenkins-hbase3.apache.org/45745 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:24:52,106 INFO [Listener at jenkins-hbase3.apache.org/45745 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:24:52,122 INFO [Listener at jenkins-hbase3.apache.org/45745 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:43799 2023-10-02T12:24:52,338 WARN [Listener at jenkins-hbase3.apache.org/37677 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:24:52,345 WARN [Listener at jenkins-hbase3.apache.org/37677 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:24:52,346 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:24:52,355 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_40113_node____.5a2pca/webapp 2023-10-02T12:24:52,417 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-10-02T12:24:52,417 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:24:52,417 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:24:52,417 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:24:52,428 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:24:52,445 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:24:52,536 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:24:52,545 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:40113 2023-10-02T12:24:52,588 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-10-02T12:24:52,588 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:36497/backupUT 2023-10-02T12:24:52,604 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:24:52,610 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35550, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:24:52,619 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-10-02T12:24:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-10-02T12:24:52,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-10-02T12:24:52,649 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:24:52,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 33 msec 2023-10-02T12:24:52,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-10-02T12:24:52,746 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-10-02T12:24:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-10-02T12:24:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-10-02T12:24:52,760 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:24:52,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 18 msec 2023-10-02T12:24:52,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-10-02T12:24:52,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-10-02T12:24:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-10-02T12:24:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-10-02T12:24:52,871 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:24:52,877 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 19 msec 2023-10-02T12:24:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-10-02T12:24:52,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-10-02T12:24:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-10-02T12:24:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-10-02T12:24:52,981 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:24:52,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 17 msec 2023-10-02T12:24:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-10-02T12:24:53,092 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'test-1696249492588', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:24:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1696249492588 2023-10-02T12:24:53,097 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1696249492588 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:24:53,098 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1696249492588" procId is: 13 2023-10-02T12:24:53,098 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:53,100 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1696249492588 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:24:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:24:53,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:24:53,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:24:53,535 INFO [RegionOpenAndInit-test-1696249492588-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => ff520da84fd90f4138d4152604b554a1, NAME => 'test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1696249492588', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a 2023-10-02T12:24:53,566 DEBUG [RegionOpenAndInit-test-1696249492588-pool-0 {}] regionserver.HRegion(888): Instantiated test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:24:53,566 DEBUG [RegionOpenAndInit-test-1696249492588-pool-0 {}] regionserver.HRegion(1668): Closing ff520da84fd90f4138d4152604b554a1, disabling compactions & flushes 2023-10-02T12:24:53,566 INFO [RegionOpenAndInit-test-1696249492588-pool-0 {}] regionserver.HRegion(1690): Closing region test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. 2023-10-02T12:24:53,566 DEBUG [RegionOpenAndInit-test-1696249492588-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. 2023-10-02T12:24:53,566 DEBUG [RegionOpenAndInit-test-1696249492588-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. after waiting 0 ms 2023-10-02T12:24:53,566 DEBUG [RegionOpenAndInit-test-1696249492588-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. 2023-10-02T12:24:53,566 INFO [RegionOpenAndInit-test-1696249492588-pool-0 {}] regionserver.HRegion(1909): Closed test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. 2023-10-02T12:24:53,567 DEBUG [RegionOpenAndInit-test-1696249492588-pool-0 {}] regionserver.HRegion(1622): Region close journal for ff520da84fd90f4138d4152604b554a1: 2023-10-02T12:24:53,568 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1696249492588 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:24:53,569 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1696249493569"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249493569"}]},"ts":"1696249493569"} 2023-10-02T12:24:53,572 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:24:53,573 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1696249492588 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:24:53,574 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696249492588","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249493573"}]},"ts":"1696249493573"} 2023-10-02T12:24:53,576 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696249492588, state=ENABLING in hbase:meta 2023-10-02T12:24:53,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1696249492588, region=ff520da84fd90f4138d4152604b554a1, ASSIGN}] 2023-10-02T12:24:53,592 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1696249492588, region=ff520da84fd90f4138d4152604b554a1, ASSIGN 2023-10-02T12:24:53,593 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1696249492588, region=ff520da84fd90f4138d4152604b554a1, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,35033,1696249482294; forceNewPlan=false, retain=false 2023-10-02T12:24:53,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:24:53,750 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=ff520da84fd90f4138d4152604b554a1, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:24:53,762 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure ff520da84fd90f4138d4152604b554a1, server=jenkins-hbase3.apache.org,35033,1696249482294}] 2023-10-02T12:24:53,924 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. 2023-10-02T12:24:53,925 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => ff520da84fd90f4138d4152604b554a1, NAME => 'test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:24:53,925 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:24:53,925 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1696249492588 ff520da84fd90f4138d4152604b554a1 2023-10-02T12:24:53,925 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:24:53,925 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for ff520da84fd90f4138d4152604b554a1 2023-10-02T12:24:53,925 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for ff520da84fd90f4138d4152604b554a1 2023-10-02T12:24:53,929 INFO [StoreOpener-ff520da84fd90f4138d4152604b554a1-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region ff520da84fd90f4138d4152604b554a1 2023-10-02T12:24:53,931 INFO [StoreOpener-ff520da84fd90f4138d4152604b554a1-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ff520da84fd90f4138d4152604b554a1 columnFamilyName f 2023-10-02T12:24:53,931 DEBUG [StoreOpener-ff520da84fd90f4138d4152604b554a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:53,932 INFO [StoreOpener-ff520da84fd90f4138d4152604b554a1-1 {}] regionserver.HStore(324): Store=ff520da84fd90f4138d4152604b554a1/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:24:53,934 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1 2023-10-02T12:24:53,934 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1 2023-10-02T12:24:53,938 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for ff520da84fd90f4138d4152604b554a1 2023-10-02T12:24:53,943 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:24:53,944 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened ff520da84fd90f4138d4152604b554a1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9643081120, jitterRate=-0.10191808640956879}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:24:53,945 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for ff520da84fd90f4138d4152604b554a1: 2023-10-02T12:24:53,946 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1., pid=15, masterSystemTime=1696249493915 2023-10-02T12:24:53,949 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. 2023-10-02T12:24:53,949 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. 2023-10-02T12:24:53,950 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=ff520da84fd90f4138d4152604b554a1, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:24:53,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-10-02T12:24:53,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure ff520da84fd90f4138d4152604b554a1, server=jenkins-hbase3.apache.org,35033,1696249482294 in 190 msec 2023-10-02T12:24:53,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-10-02T12:24:53,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1696249492588, region=ff520da84fd90f4138d4152604b554a1, ASSIGN in 367 msec 2023-10-02T12:24:53,959 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1696249492588 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:24:53,959 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696249492588","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249493959"}]},"ts":"1696249493959"} 2023-10-02T12:24:53,961 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696249492588, state=ENABLED in hbase:meta 2023-10-02T12:24:53,964 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1696249492588 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:24:53,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1696249492588 in 872 msec 2023-10-02T12:24:54,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:24:54,215 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:test-1696249492588, procId: 13 completed 2023-10-02T12:24:54,216 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x566748b1 to 127.0.0.1:58172 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:24:54,224 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@665c82e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:24:54,226 DEBUG [hconnection-0x284bcbfe-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:24:54,230 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46880, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:24:54,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35033 {}] regionserver.HRegion(8210): writing data to region test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. with WAL disabled. Data may be lost in the event of a crash. 2023-10-02T12:24:54,352 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns2:test-16962494925881', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:24:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16962494925881 2023-10-02T12:24:54,358 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16962494925881 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:24:54,358 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:54,358 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16962494925881" procId is: 16 2023-10-02T12:24:54,359 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16962494925881 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:24:54,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:24:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:24:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:24:54,784 INFO [RegionOpenAndInit-ns2:test-16962494925881-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => a39f2861eedc2f7c63f0bcd60085007e, NAME => 'ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16962494925881', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a 2023-10-02T12:24:54,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:24:55,202 DEBUG [RegionOpenAndInit-ns2:test-16962494925881-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:24:55,202 DEBUG [RegionOpenAndInit-ns2:test-16962494925881-pool-0 {}] regionserver.HRegion(1668): Closing a39f2861eedc2f7c63f0bcd60085007e, disabling compactions & flushes 2023-10-02T12:24:55,202 INFO [RegionOpenAndInit-ns2:test-16962494925881-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e. 2023-10-02T12:24:55,202 DEBUG [RegionOpenAndInit-ns2:test-16962494925881-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e. 2023-10-02T12:24:55,202 DEBUG [RegionOpenAndInit-ns2:test-16962494925881-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e. after waiting 0 ms 2023-10-02T12:24:55,202 DEBUG [RegionOpenAndInit-ns2:test-16962494925881-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e. 2023-10-02T12:24:55,203 INFO [RegionOpenAndInit-ns2:test-16962494925881-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e. 2023-10-02T12:24:55,203 DEBUG [RegionOpenAndInit-ns2:test-16962494925881-pool-0 {}] regionserver.HRegion(1622): Region close journal for a39f2861eedc2f7c63f0bcd60085007e: 2023-10-02T12:24:55,204 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16962494925881 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:24:55,205 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696249495205"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249495205"}]},"ts":"1696249495205"} 2023-10-02T12:24:55,208 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:24:55,209 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16962494925881 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:24:55,209 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962494925881","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249495209"}]},"ts":"1696249495209"} 2023-10-02T12:24:55,212 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962494925881, state=ENABLING in hbase:meta 2023-10-02T12:24:55,217 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16962494925881, region=a39f2861eedc2f7c63f0bcd60085007e, ASSIGN}] 2023-10-02T12:24:55,220 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16962494925881, region=a39f2861eedc2f7c63f0bcd60085007e, ASSIGN 2023-10-02T12:24:55,221 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16962494925881, region=a39f2861eedc2f7c63f0bcd60085007e, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,35033,1696249482294; forceNewPlan=false, retain=false 2023-10-02T12:24:55,373 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=a39f2861eedc2f7c63f0bcd60085007e, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:24:55,376 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure a39f2861eedc2f7c63f0bcd60085007e, server=jenkins-hbase3.apache.org,35033,1696249482294}] 2023-10-02T12:24:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:24:55,535 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e. 2023-10-02T12:24:55,536 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => a39f2861eedc2f7c63f0bcd60085007e, NAME => 'ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:24:55,536 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:24:55,536 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962494925881 a39f2861eedc2f7c63f0bcd60085007e 2023-10-02T12:24:55,537 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:24:55,537 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for a39f2861eedc2f7c63f0bcd60085007e 2023-10-02T12:24:55,537 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for a39f2861eedc2f7c63f0bcd60085007e 2023-10-02T12:24:55,539 INFO [StoreOpener-a39f2861eedc2f7c63f0bcd60085007e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region a39f2861eedc2f7c63f0bcd60085007e 2023-10-02T12:24:55,542 INFO [StoreOpener-a39f2861eedc2f7c63f0bcd60085007e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a39f2861eedc2f7c63f0bcd60085007e columnFamilyName f 2023-10-02T12:24:55,542 DEBUG [StoreOpener-a39f2861eedc2f7c63f0bcd60085007e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:55,543 INFO [StoreOpener-a39f2861eedc2f7c63f0bcd60085007e-1 {}] regionserver.HStore(324): Store=a39f2861eedc2f7c63f0bcd60085007e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:24:55,544 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/ns2/test-16962494925881/a39f2861eedc2f7c63f0bcd60085007e 2023-10-02T12:24:55,544 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/ns2/test-16962494925881/a39f2861eedc2f7c63f0bcd60085007e 2023-10-02T12:24:55,548 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for a39f2861eedc2f7c63f0bcd60085007e 2023-10-02T12:24:55,561 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/ns2/test-16962494925881/a39f2861eedc2f7c63f0bcd60085007e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:24:55,563 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened a39f2861eedc2f7c63f0bcd60085007e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11100518400, jitterRate=0.03381633758544922}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:24:55,564 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for a39f2861eedc2f7c63f0bcd60085007e: 2023-10-02T12:24:55,566 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e., pid=18, masterSystemTime=1696249495530 2023-10-02T12:24:55,568 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e. 2023-10-02T12:24:55,569 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e. 2023-10-02T12:24:55,570 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=a39f2861eedc2f7c63f0bcd60085007e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:24:55,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-10-02T12:24:55,576 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure a39f2861eedc2f7c63f0bcd60085007e, server=jenkins-hbase3.apache.org,35033,1696249482294 in 196 msec 2023-10-02T12:24:55,579 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-10-02T12:24:55,579 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16962494925881, region=a39f2861eedc2f7c63f0bcd60085007e, ASSIGN in 359 msec 2023-10-02T12:24:55,580 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16962494925881 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:24:55,581 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962494925881","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249495580"}]},"ts":"1696249495580"} 2023-10-02T12:24:55,583 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962494925881, state=ENABLED in hbase:meta 2023-10-02T12:24:55,587 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16962494925881 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:24:55,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16962494925881 in 1.2340 sec 2023-10-02T12:24:56,354 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1696249492588' 2023-10-02T12:24:56,355 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16962494925881' 2023-10-02T12:24:56,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:24:56,472 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns2:test-16962494925881, procId: 16 completed 2023-10-02T12:24:56,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35033 {}] regionserver.HRegion(8210): writing data to region ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e. with WAL disabled. Data may be lost in the event of a crash. 2023-10-02T12:24:56,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns3:test-16962494925882', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:24:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16962494925882 2023-10-02T12:24:56,602 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16962494925882 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:24:56,602 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:56,602 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16962494925882" procId is: 19 2023-10-02T12:24:56,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:24:56,605 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16962494925882 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:24:56,630 INFO [RegionOpenAndInit-ns3:test-16962494925882-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => a09bef7b244cd13ac616d106a9188033, NAME => 'ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16962494925882', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a 2023-10-02T12:24:56,655 DEBUG [RegionOpenAndInit-ns3:test-16962494925882-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:24:56,655 DEBUG [RegionOpenAndInit-ns3:test-16962494925882-pool-0 {}] regionserver.HRegion(1668): Closing a09bef7b244cd13ac616d106a9188033, disabling compactions & flushes 2023-10-02T12:24:56,655 INFO [RegionOpenAndInit-ns3:test-16962494925882-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033. 2023-10-02T12:24:56,656 DEBUG [RegionOpenAndInit-ns3:test-16962494925882-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033. 2023-10-02T12:24:56,656 DEBUG [RegionOpenAndInit-ns3:test-16962494925882-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033. after waiting 0 ms 2023-10-02T12:24:56,656 DEBUG [RegionOpenAndInit-ns3:test-16962494925882-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033. 2023-10-02T12:24:56,656 INFO [RegionOpenAndInit-ns3:test-16962494925882-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033. 2023-10-02T12:24:56,656 DEBUG [RegionOpenAndInit-ns3:test-16962494925882-pool-0 {}] regionserver.HRegion(1622): Region close journal for a09bef7b244cd13ac616d106a9188033: 2023-10-02T12:24:56,658 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16962494925882 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:24:56,658 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696249496658"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249496658"}]},"ts":"1696249496658"} 2023-10-02T12:24:56,661 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:24:56,663 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16962494925882 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:24:56,663 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962494925882","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249496663"}]},"ts":"1696249496663"} 2023-10-02T12:24:56,668 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962494925882, state=ENABLING in hbase:meta 2023-10-02T12:24:56,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16962494925882, region=a09bef7b244cd13ac616d106a9188033, ASSIGN}] 2023-10-02T12:24:56,677 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16962494925882, region=a09bef7b244cd13ac616d106a9188033, ASSIGN 2023-10-02T12:24:56,679 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16962494925882, region=a09bef7b244cd13ac616d106a9188033, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,35033,1696249482294; forceNewPlan=false, retain=false 2023-10-02T12:24:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:24:56,831 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=a09bef7b244cd13ac616d106a9188033, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:24:56,834 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure a09bef7b244cd13ac616d106a9188033, server=jenkins-hbase3.apache.org,35033,1696249482294}] 2023-10-02T12:24:56,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:24:56,992 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033. 2023-10-02T12:24:56,993 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => a09bef7b244cd13ac616d106a9188033, NAME => 'ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:24:56,993 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:24:56,994 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962494925882 a09bef7b244cd13ac616d106a9188033 2023-10-02T12:24:56,994 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:24:56,994 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for a09bef7b244cd13ac616d106a9188033 2023-10-02T12:24:56,994 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for a09bef7b244cd13ac616d106a9188033 2023-10-02T12:24:56,996 INFO [StoreOpener-a09bef7b244cd13ac616d106a9188033-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region a09bef7b244cd13ac616d106a9188033 2023-10-02T12:24:56,998 INFO [StoreOpener-a09bef7b244cd13ac616d106a9188033-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a09bef7b244cd13ac616d106a9188033 columnFamilyName f 2023-10-02T12:24:56,998 DEBUG [StoreOpener-a09bef7b244cd13ac616d106a9188033-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:56,999 INFO [StoreOpener-a09bef7b244cd13ac616d106a9188033-1 {}] regionserver.HStore(324): Store=a09bef7b244cd13ac616d106a9188033/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:24:57,000 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/ns3/test-16962494925882/a09bef7b244cd13ac616d106a9188033 2023-10-02T12:24:57,001 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/ns3/test-16962494925882/a09bef7b244cd13ac616d106a9188033 2023-10-02T12:24:57,005 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for a09bef7b244cd13ac616d106a9188033 2023-10-02T12:24:57,013 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/ns3/test-16962494925882/a09bef7b244cd13ac616d106a9188033/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:24:57,014 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened a09bef7b244cd13ac616d106a9188033; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9941916160, jitterRate=-0.07408690452575684}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:24:57,014 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for a09bef7b244cd13ac616d106a9188033: 2023-10-02T12:24:57,016 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033., pid=21, masterSystemTime=1696249496987 2023-10-02T12:24:57,019 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033. 2023-10-02T12:24:57,019 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033. 2023-10-02T12:24:57,020 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=a09bef7b244cd13ac616d106a9188033, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:24:57,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-10-02T12:24:57,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure a09bef7b244cd13ac616d106a9188033, server=jenkins-hbase3.apache.org,35033,1696249482294 in 188 msec 2023-10-02T12:24:57,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-10-02T12:24:57,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16962494925882, region=a09bef7b244cd13ac616d106a9188033, ASSIGN in 352 msec 2023-10-02T12:24:57,029 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16962494925882 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:24:57,029 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962494925882","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249497029"}]},"ts":"1696249497029"} 2023-10-02T12:24:57,031 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962494925882, state=ENABLED in hbase:meta 2023-10-02T12:24:57,034 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16962494925882 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:24:57,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16962494925882 in 438 msec 2023-10-02T12:24:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:24:57,213 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns3:test-16962494925882, procId: 19 completed 2023-10-02T12:24:57,213 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16962494925882 get assigned. Timeout = 60000ms 2023-10-02T12:24:57,215 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:24:57,219 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16962494925882 assigned to meta. Checking AM states. 2023-10-02T12:24:57,220 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:24:57,220 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16962494925882 assigned. 2023-10-02T12:24:57,223 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns4:test-16962494925883', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:24:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16962494925883 2023-10-02T12:24:57,227 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16962494925883 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:24:57,227 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:57,227 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16962494925883" procId is: 22 2023-10-02T12:24:57,228 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16962494925883 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:24:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:24:57,243 INFO [RegionOpenAndInit-ns4:test-16962494925883-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 9ce225be3bc8c4e3b577db51ccdc0ec7, NAME => 'ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16962494925883', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a 2023-10-02T12:24:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:24:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:24:57,659 DEBUG [RegionOpenAndInit-ns4:test-16962494925883-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:24:57,659 DEBUG [RegionOpenAndInit-ns4:test-16962494925883-pool-0 {}] regionserver.HRegion(1668): Closing 9ce225be3bc8c4e3b577db51ccdc0ec7, disabling compactions & flushes 2023-10-02T12:24:57,659 INFO [RegionOpenAndInit-ns4:test-16962494925883-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7. 2023-10-02T12:24:57,659 DEBUG [RegionOpenAndInit-ns4:test-16962494925883-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7. 2023-10-02T12:24:57,659 DEBUG [RegionOpenAndInit-ns4:test-16962494925883-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7. after waiting 0 ms 2023-10-02T12:24:57,659 DEBUG [RegionOpenAndInit-ns4:test-16962494925883-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7. 2023-10-02T12:24:57,659 INFO [RegionOpenAndInit-ns4:test-16962494925883-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7. 2023-10-02T12:24:57,659 DEBUG [RegionOpenAndInit-ns4:test-16962494925883-pool-0 {}] regionserver.HRegion(1622): Region close journal for 9ce225be3bc8c4e3b577db51ccdc0ec7: 2023-10-02T12:24:57,661 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16962494925883 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:24:57,662 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696249497661"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249497661"}]},"ts":"1696249497661"} 2023-10-02T12:24:57,664 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:24:57,665 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16962494925883 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:24:57,666 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16962494925883","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249497665"}]},"ts":"1696249497665"} 2023-10-02T12:24:57,668 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16962494925883, state=ENABLING in hbase:meta 2023-10-02T12:24:57,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16962494925883, region=9ce225be3bc8c4e3b577db51ccdc0ec7, ASSIGN}] 2023-10-02T12:24:57,686 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16962494925883, region=9ce225be3bc8c4e3b577db51ccdc0ec7, ASSIGN 2023-10-02T12:24:57,688 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16962494925883, region=9ce225be3bc8c4e3b577db51ccdc0ec7, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,35033,1696249482294; forceNewPlan=false, retain=false 2023-10-02T12:24:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:24:57,839 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=9ce225be3bc8c4e3b577db51ccdc0ec7, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:24:57,842 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 9ce225be3bc8c4e3b577db51ccdc0ec7, server=jenkins-hbase3.apache.org,35033,1696249482294}] 2023-10-02T12:24:58,000 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7. 2023-10-02T12:24:58,001 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 9ce225be3bc8c4e3b577db51ccdc0ec7, NAME => 'ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:24:58,001 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:24:58,001 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962494925883 9ce225be3bc8c4e3b577db51ccdc0ec7 2023-10-02T12:24:58,002 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:24:58,002 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 9ce225be3bc8c4e3b577db51ccdc0ec7 2023-10-02T12:24:58,002 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 9ce225be3bc8c4e3b577db51ccdc0ec7 2023-10-02T12:24:58,004 INFO [StoreOpener-9ce225be3bc8c4e3b577db51ccdc0ec7-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 9ce225be3bc8c4e3b577db51ccdc0ec7 2023-10-02T12:24:58,007 INFO [StoreOpener-9ce225be3bc8c4e3b577db51ccdc0ec7-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9ce225be3bc8c4e3b577db51ccdc0ec7 columnFamilyName f 2023-10-02T12:24:58,007 DEBUG [StoreOpener-9ce225be3bc8c4e3b577db51ccdc0ec7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:58,007 INFO [StoreOpener-9ce225be3bc8c4e3b577db51ccdc0ec7-1 {}] regionserver.HStore(324): Store=9ce225be3bc8c4e3b577db51ccdc0ec7/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:24:58,009 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/ns4/test-16962494925883/9ce225be3bc8c4e3b577db51ccdc0ec7 2023-10-02T12:24:58,009 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/ns4/test-16962494925883/9ce225be3bc8c4e3b577db51ccdc0ec7 2023-10-02T12:24:58,013 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 9ce225be3bc8c4e3b577db51ccdc0ec7 2023-10-02T12:24:58,017 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/ns4/test-16962494925883/9ce225be3bc8c4e3b577db51ccdc0ec7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:24:58,018 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 9ce225be3bc8c4e3b577db51ccdc0ec7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11657333440, jitterRate=0.08567377924919128}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:24:58,019 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 9ce225be3bc8c4e3b577db51ccdc0ec7: 2023-10-02T12:24:58,020 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7., pid=24, masterSystemTime=1696249497995 2023-10-02T12:24:58,023 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7. 2023-10-02T12:24:58,023 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7. 2023-10-02T12:24:58,024 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=9ce225be3bc8c4e3b577db51ccdc0ec7, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:24:58,028 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-10-02T12:24:58,028 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 9ce225be3bc8c4e3b577db51ccdc0ec7, server=jenkins-hbase3.apache.org,35033,1696249482294 in 184 msec 2023-10-02T12:24:58,031 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-10-02T12:24:58,031 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16962494925883, region=9ce225be3bc8c4e3b577db51ccdc0ec7, ASSIGN in 345 msec 2023-10-02T12:24:58,031 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16962494925883 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:24:58,032 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16962494925883","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249498032"}]},"ts":"1696249498032"} 2023-10-02T12:24:58,033 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16962494925883, state=ENABLED in hbase:meta 2023-10-02T12:24:58,036 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16962494925883 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:24:58,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16962494925883 in 813 msec 2023-10-02T12:24:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:24:58,337 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns4:test-16962494925883, procId: 22 completed 2023-10-02T12:24:58,337 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16962494925883 get assigned. Timeout = 60000ms 2023-10-02T12:24:58,337 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:24:58,343 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16962494925883 assigned to meta. Checking AM states. 2023-10-02T12:24:58,343 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:24:58,343 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16962494925883 assigned. 2023-10-02T12:24:58,344 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x566748b1 to 127.0.0.1:58172 2023-10-02T12:24:58,344 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:24:58,395 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] hbase.ResourceChecker(147): before: backup.TestBackupDeleteWithFailures#testBackupDeleteWithFailures Thread=583, OpenFileDescriptor=660, MaxFileDescriptor=60000, SystemLoadAverage=522, ProcessCount=183, AvailableMemoryMB=2496 2023-10-02T12:24:58,395 WARN [Listener at jenkins-hbase3.apache.org/37677 {}] hbase.ResourceChecker(130): Thread=583 is superior to 500 2023-10-02T12:24:58,398 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] backup.TestBackupDeleteWithFailures(143): test repair backup delete on a single table with data and failures PRE_DELETE_SNAPSHOT_FAILURE 2023-10-02T12:24:58,400 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x70409664 to 127.0.0.1:58172 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:24:58,456 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ffb251f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:24:58,467 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:24:58,468 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46890, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:24:58,482 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:24:58,485 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:35554, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:24:58,507 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-10-02T12:24:58,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-10-02T12:24:58,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-10-02T12:24:58,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-10-02T12:24:58,747 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:24:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-10-02T12:24:58,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 335 msec 2023-10-02T12:24:59,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-10-02T12:24:59,137 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:24:59,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-10-02T12:24:59,142 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:24:59,142 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:24:59,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-10-02T12:24:59,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:24:59,144 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:24:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:24:59,444 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:24:59,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:24:59,570 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 3dfc58254ae9ba5ba94c6dcd551aba30, NAME => 'backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a 2023-10-02T12:24:59,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:24:59,994 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:24:59,995 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 3dfc58254ae9ba5ba94c6dcd551aba30, disabling compactions & flushes 2023-10-02T12:24:59,995 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:24:59,995 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:24:59,995 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. after waiting 0 ms 2023-10-02T12:24:59,995 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:24:59,995 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:24:59,995 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 3dfc58254ae9ba5ba94c6dcd551aba30: 2023-10-02T12:24:59,997 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:24:59,998 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1696249499997"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249499997"}]},"ts":"1696249499997"} 2023-10-02T12:25:00,000 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:25:00,002 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:25:00,002 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249500002"}]},"ts":"1696249500002"} 2023-10-02T12:25:00,005 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-10-02T12:25:00,051 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16962494925882 2023-10-02T12:25:00,051 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16962494925882 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:25:00,052 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16962494925881 2023-10-02T12:25:00,052 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16962494925881 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:25:00,053 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16962494925883 2023-10-02T12:25:00,053 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16962494925883 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:25:00,054 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1696249492588 2023-10-02T12:25:00,054 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1696249492588 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:25:00,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=3dfc58254ae9ba5ba94c6dcd551aba30, ASSIGN}] 2023-10-02T12:25:00,167 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=3dfc58254ae9ba5ba94c6dcd551aba30, ASSIGN 2023-10-02T12:25:00,168 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=3dfc58254ae9ba5ba94c6dcd551aba30, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,35033,1696249482294; forceNewPlan=false, retain=false 2023-10-02T12:25:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:25:00,320 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=3dfc58254ae9ba5ba94c6dcd551aba30, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:00,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30, server=jenkins-hbase3.apache.org,35033,1696249482294}] 2023-10-02T12:25:00,481 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:00,482 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => 3dfc58254ae9ba5ba94c6dcd551aba30, NAME => 'backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:25:00,483 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:25:00,483 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:00,483 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:25:00,483 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:00,483 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:00,485 INFO [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:00,488 INFO [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3dfc58254ae9ba5ba94c6dcd551aba30 columnFamilyName meta 2023-10-02T12:25:00,488 DEBUG [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:25:00,489 INFO [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] regionserver.HStore(324): Store=3dfc58254ae9ba5ba94c6dcd551aba30/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:25:00,489 INFO [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:00,491 INFO [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3dfc58254ae9ba5ba94c6dcd551aba30 columnFamilyName session 2023-10-02T12:25:00,491 DEBUG [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:25:00,492 INFO [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] regionserver.HStore(324): Store=3dfc58254ae9ba5ba94c6dcd551aba30/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:25:00,493 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:00,494 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:00,495 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:25:00,497 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:00,500 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:25:00,501 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened 3dfc58254ae9ba5ba94c6dcd551aba30; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11945793120, jitterRate=0.11253868043422699}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:25:00,501 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for 3dfc58254ae9ba5ba94c6dcd551aba30: 2023-10-02T12:25:00,503 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30., pid=28, masterSystemTime=1696249500476 2023-10-02T12:25:00,505 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:00,505 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:00,506 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=3dfc58254ae9ba5ba94c6dcd551aba30, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:00,511 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-10-02T12:25:00,511 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30, server=jenkins-hbase3.apache.org,35033,1696249482294 in 185 msec 2023-10-02T12:25:00,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-10-02T12:25:00,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=3dfc58254ae9ba5ba94c6dcd551aba30, ASSIGN in 347 msec 2023-10-02T12:25:00,515 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:25:00,515 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249500515"}]},"ts":"1696249500515"} 2023-10-02T12:25:00,517 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-10-02T12:25:00,521 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:25:00,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 1.3840 sec 2023-10-02T12:25:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:25:01,260 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-10-02T12:25:01,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:25:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-10-02T12:25:01,287 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:25:01,287 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:25:01,287 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-10-02T12:25:01,289 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:25:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:25:01,326 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 0321d0b0338f45d0ba3fea553dbc434d, NAME => 'backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a 2023-10-02T12:25:01,342 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:25:01,342 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing 0321d0b0338f45d0ba3fea553dbc434d, disabling compactions & flushes 2023-10-02T12:25:01,342 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d. 2023-10-02T12:25:01,342 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d. 2023-10-02T12:25:01,342 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d. after waiting 0 ms 2023-10-02T12:25:01,343 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d. 2023-10-02T12:25:01,343 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d. 2023-10-02T12:25:01,343 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for 0321d0b0338f45d0ba3fea553dbc434d: 2023-10-02T12:25:01,344 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:25:01,345 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1696249501344"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249501344"}]},"ts":"1696249501344"} 2023-10-02T12:25:01,347 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:25:01,348 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:25:01,349 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249501349"}]},"ts":"1696249501349"} 2023-10-02T12:25:01,351 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-10-02T12:25:01,358 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=0321d0b0338f45d0ba3fea553dbc434d, ASSIGN}] 2023-10-02T12:25:01,361 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=0321d0b0338f45d0ba3fea553dbc434d, ASSIGN 2023-10-02T12:25:01,362 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=0321d0b0338f45d0ba3fea553dbc434d, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,35033,1696249482294; forceNewPlan=false, retain=false 2023-10-02T12:25:01,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:25:01,514 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=0321d0b0338f45d0ba3fea553dbc434d, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:01,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure 0321d0b0338f45d0ba3fea553dbc434d, server=jenkins-hbase3.apache.org,35033,1696249482294}] 2023-10-02T12:25:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:25:01,676 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d. 2023-10-02T12:25:01,677 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => 0321d0b0338f45d0ba3fea553dbc434d, NAME => 'backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:25:01,677 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:25:01,677 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 0321d0b0338f45d0ba3fea553dbc434d 2023-10-02T12:25:01,677 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:25:01,678 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for 0321d0b0338f45d0ba3fea553dbc434d 2023-10-02T12:25:01,678 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for 0321d0b0338f45d0ba3fea553dbc434d 2023-10-02T12:25:01,683 INFO [StoreOpener-0321d0b0338f45d0ba3fea553dbc434d-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 0321d0b0338f45d0ba3fea553dbc434d 2023-10-02T12:25:01,686 INFO [StoreOpener-0321d0b0338f45d0ba3fea553dbc434d-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0321d0b0338f45d0ba3fea553dbc434d columnFamilyName meta 2023-10-02T12:25:01,686 DEBUG [StoreOpener-0321d0b0338f45d0ba3fea553dbc434d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:25:01,687 INFO [StoreOpener-0321d0b0338f45d0ba3fea553dbc434d-1 {}] regionserver.HStore(324): Store=0321d0b0338f45d0ba3fea553dbc434d/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:25:01,687 INFO [StoreOpener-0321d0b0338f45d0ba3fea553dbc434d-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 0321d0b0338f45d0ba3fea553dbc434d 2023-10-02T12:25:01,689 INFO [StoreOpener-0321d0b0338f45d0ba3fea553dbc434d-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0321d0b0338f45d0ba3fea553dbc434d columnFamilyName session 2023-10-02T12:25:01,689 DEBUG [StoreOpener-0321d0b0338f45d0ba3fea553dbc434d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:25:01,690 INFO [StoreOpener-0321d0b0338f45d0ba3fea553dbc434d-1 {}] regionserver.HStore(324): Store=0321d0b0338f45d0ba3fea553dbc434d/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:25:01,691 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system_bulk/0321d0b0338f45d0ba3fea553dbc434d 2023-10-02T12:25:01,692 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system_bulk/0321d0b0338f45d0ba3fea553dbc434d 2023-10-02T12:25:01,694 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:25:01,696 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for 0321d0b0338f45d0ba3fea553dbc434d 2023-10-02T12:25:01,700 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system_bulk/0321d0b0338f45d0ba3fea553dbc434d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:25:01,701 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened 0321d0b0338f45d0ba3fea553dbc434d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10936903520, jitterRate=0.018578514456748962}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:25:01,702 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for 0321d0b0338f45d0ba3fea553dbc434d: 2023-10-02T12:25:01,703 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d., pid=31, masterSystemTime=1696249501671 2023-10-02T12:25:01,711 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d. 2023-10-02T12:25:01,712 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d. 2023-10-02T12:25:01,712 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=0321d0b0338f45d0ba3fea553dbc434d, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:01,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-10-02T12:25:01,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure 0321d0b0338f45d0ba3fea553dbc434d, server=jenkins-hbase3.apache.org,35033,1696249482294 in 197 msec 2023-10-02T12:25:01,730 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=30, resume processing ppid=29 2023-10-02T12:25:01,730 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=0321d0b0338f45d0ba3fea553dbc434d, ASSIGN in 360 msec 2023-10-02T12:25:01,732 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:25:01,732 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249501732"}]},"ts":"1696249501732"} 2023-10-02T12:25:01,736 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-10-02T12:25:01,739 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:25:01,742 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 462 msec 2023-10-02T12:25:01,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:25:01,896 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-10-02T12:25:01,912 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:25:01,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:25:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249501967 (current time:1696249501967). 2023-10-02T12:25:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:25:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:25:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x110ad893 to 127.0.0.1:58172 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:25:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fb692eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:25:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:25:01,983 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34390, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:25:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x110ad893 to 127.0.0.1:58172 2023-10-02T12:25:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:01,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:25:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:25:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-10-02T12:25:02,004 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:25:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:25:02,012 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:25:02,029 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot, current working dir path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:25:02,038 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:25:02,055 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:25:02,058 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30}] 2023-10-02T12:25:02,065 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:25:02,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35033 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-10-02T12:25:02,228 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:02,230 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing 3dfc58254ae9ba5ba94c6dcd551aba30 2/2 column families, dataSize=45 B heapSize=632 B 2023-10-02T12:25:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:25:02,324 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/9f3d5d24e61b47dca538b66fb59a927a is 49, key is activesession:/session:c/1696249501951/Put/seqid=0 2023-10-02T12:25:02,356 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-10-02T12:25:02,357 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16962494925882' 2023-10-02T12:25:02,358 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16962494925883' 2023-10-02T12:25:02,358 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-10-02T12:25:02,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:25:02,749 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/9f3d5d24e61b47dca538b66fb59a927a 2023-10-02T12:25:02,864 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/9f3d5d24e61b47dca538b66fb59a927a as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/9f3d5d24e61b47dca538b66fb59a927a 2023-10-02T12:25:02,884 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/9f3d5d24e61b47dca538b66fb59a927a, entries=1, sequenceid=5, filesize=4.9 K 2023-10-02T12:25:02,889 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 3dfc58254ae9ba5ba94c6dcd551aba30 in 659ms, sequenceid=5, compaction requested=false 2023-10-02T12:25:02,892 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for 3dfc58254ae9ba5ba94c6dcd551aba30: 2023-10-02T12:25:02,892 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. for snapshot_backup_system completed. 2023-10-02T12:25:02,894 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:25:02,903 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:25:02,910 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:25:02,910 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/9f3d5d24e61b47dca538b66fb59a927a] hfiles 2023-10-02T12:25:02,910 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/9f3d5d24e61b47dca538b66fb59a927a for snapshot=snapshot_backup_system 2023-10-02T12:25:02,938 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:02,942 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-10-02T12:25:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster(3995): Remote procedure done, pid=33 2023-10-02T12:25:02,946 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:02,947 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:02,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-10-02T12:25:02,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30 in 890 msec 2023-10-02T12:25:02,951 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:25:02,954 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:25:02,962 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:25:02,962 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:25:02,966 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:25:03,003 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:25:03,022 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:25:03,023 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:25:03,026 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:25:03,026 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-10-02T12:25:03,028 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.0310 sec 2023-10-02T12:25:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:25:03,115 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-10-02T12:25:03,115 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(115): Backup backup_1696249498461 started at 1696249503115. 2023-10-02T12:25:03,142 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(119): Backup session backup_1696249498461 has been started. 2023-10-02T12:25:03,147 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:25:03,166 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:25:03,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:25:03,179 INFO [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:25:03,179 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:25:03,180 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:25:03,180 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:25:03,181 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:25:03,181 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:25:03,184 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:25:03,184 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:03,184 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:25:03,185 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:25:03,185 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:25:03,186 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:25:03,186 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:03,186 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:25:03,186 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:25:03,186 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:25:03,187 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:25:03,189 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:25:03,189 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:25:03,189 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:25:03,190 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:25:03,191 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:25:03,191 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:25:03,191 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,35033,1696249482294' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:25:03,194 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:03,195 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:03,195 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:25:03,195 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:25:03,195 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:25:03,196 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:25:03,196 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:25:03,197 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-10-02T12:25:03,197 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:25:03,197 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:25:03,198 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:03,198 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:25:03,199 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,35033,1696249482294' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:25:03,199 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@6ab77426[Count = 0] remaining members to acquire global barrier 2023-10-02T12:25:03,199 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:25:03,199 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:25:03,202 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:25:03,202 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:25:03,202 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:25:03,202 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:25:03,203 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:03,203 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:25:03,210 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:25:03,214 DEBUG [rs(jenkins-hbase3.apache.org,35033,1696249482294)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:03,214 INFO [rs(jenkins-hbase3.apache.org,35033,1696249482294)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696249484548 highest: 1696249484548 on jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:03,214 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C35033%2C1696249482294:(num 1696249484548) roll requested 2023-10-02T12:25:03,231 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.1696249503214, exclude list is [], retry=0 2023-10-02T12:25:03,251 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38869,DS-d3b004e5-e24b-4408-a9bf-421ca1308ded,DISK] 2023-10-02T12:25:03,263 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.1696249484548 with entries=19, filesize=3.99 KB; new WAL /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.1696249503214 2023-10-02T12:25:03,264 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38869,DS-d3b004e5-e24b-4408-a9bf-421ca1308ded,DISK]] 2023-10-02T12:25:03,264 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.1696249484548 is not closed yet, will try archiving it next time 2023-10-02T12:25:03,264 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta:.meta(num 1696249485541) roll requested 2023-10-02T12:25:03,280 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta.1696249503264.meta, exclude list is [], retry=0 2023-10-02T12:25:03,288 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38869,DS-d3b004e5-e24b-4408-a9bf-421ca1308ded,DISK] 2023-10-02T12:25:03,292 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta.1696249485541.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta.1696249503264.meta 2023-10-02T12:25:03,292 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38869,DS-d3b004e5-e24b-4408-a9bf-421ca1308ded,DISK]] 2023-10-02T12:25:03,292 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta.1696249485541.meta is not closed yet, will try archiving it next time 2023-10-02T12:25:03,296 DEBUG [rs(jenkins-hbase3.apache.org,35033,1696249482294)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 82 2023-10-02T12:25:03,296 INFO [rs(jenkins-hbase3.apache.org,35033,1696249482294)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696249503214 on jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:03,297 DEBUG [rs(jenkins-hbase3.apache.org,35033,1696249482294)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:25:03,300 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57690, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-10-02T12:25:03,325 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:25:03,325 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,35033,1696249482294' in zk 2023-10-02T12:25:03,329 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:25:03,329 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:03,329 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:25:03,329 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:03,329 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:25:03,330 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:25:03,330 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:25:03,331 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:25:03,331 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:25:03,332 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:25:03,332 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:03,333 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:25:03,333 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:25:03,334 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:03,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,35033,1696249482294': 2023-10-02T12:25:03,335 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,35033,1696249482294' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:25:03,335 INFO [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:25:03,335 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:25:03,335 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:25:03,335 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:25:03,335 INFO [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:25:03,340 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:25:03,340 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:25:03,340 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:25:03,340 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:25:03,340 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:25:03,340 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:25:03,340 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:25:03,340 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:25:03,340 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:25:03,340 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:25:03,341 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:25:03,341 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:03,341 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:25:03,341 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:25:03,342 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:25:03,342 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:25:03,343 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:03,343 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:03,344 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:25:03,344 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:25:03,347 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:03,353 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:25:03,353 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:25:03,353 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): regionserver:35033-0x10173225a500001, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:25:03,353 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:03,353 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:25:03,353 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:25:03,353 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:25:03,353 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:03,353 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:25:03,353 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:25:03,353 DEBUG [Listener at localhost/43179-EventThread {}] zookeeper.ZKWatcher(604): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:25:03,353 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:25:03,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:25:03,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:25:03,354 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:25:03,354 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:25:03,355 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:43611-0x10173225a500000, quorum=127.0.0.1:58172, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-10-02T12:25:03,358 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 100000 ms per retry) 2023-10-02T12:25:03,358 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-10-02T12:25:03,459 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-10-02T12:25:03,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:25:03,473 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249503472_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } 2023-10-02T12:25:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249503473 (current time:1696249503473). 2023-10-02T12:25:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:25:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249503472_default_test-1696249492588 VERSION not specified, setting to 2 2023-10-02T12:25:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7e084061 to 127.0.0.1:58172 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:25:03,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46221ea6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:25:03,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:25:03,489 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34402, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:25:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7e084061 to 127.0.0.1:58172 2023-10-02T12:25:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:25:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249503472_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } 2023-10-02T12:25:03,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249503472_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-10-02T12:25:03,495 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249503472_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:25:03,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:25:03,497 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249503472_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:25:03,501 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot, current working dir path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_1696249503472_default_test-1696249492588 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:25:03,508 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249503472_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:25:03,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:25:03,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:25:03,938 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249503472_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:25:03,938 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure ff520da84fd90f4138d4152604b554a1}] 2023-10-02T12:25:03,940 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure ff520da84fd90f4138d4152604b554a1 2023-10-02T12:25:04,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35033 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-10-02T12:25:04,093 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. 2023-10-02T12:25:04,095 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing ff520da84fd90f4138d4152604b554a1 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-10-02T12:25:04,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:25:04,119 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1/.tmp/f/fdcfc2b4f61244028e163a86e4ae5481 is 37, key is row10/f:q1/1696249494249/Put/seqid=0 2023-10-02T12:25:04,550 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1/.tmp/f/fdcfc2b4f61244028e163a86e4ae5481 2023-10-02T12:25:04,564 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1/.tmp/f/fdcfc2b4f61244028e163a86e4ae5481 as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1/f/fdcfc2b4f61244028e163a86e4ae5481 2023-10-02T12:25:04,574 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1/f/fdcfc2b4f61244028e163a86e4ae5481, entries=99, sequenceid=103, filesize=8.2 K 2023-10-02T12:25:04,576 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for ff520da84fd90f4138d4152604b554a1 in 481ms, sequenceid=103, compaction requested=false 2023-10-02T12:25:04,576 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for ff520da84fd90f4138d4152604b554a1: 2023-10-02T12:25:04,577 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. for snapshot_1696249503472_default_test-1696249492588 completed. 2023-10-02T12:25:04,577 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1.' region-info for snapshot=snapshot_1696249503472_default_test-1696249492588 2023-10-02T12:25:04,577 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:25:04,577 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1/f/fdcfc2b4f61244028e163a86e4ae5481] hfiles 2023-10-02T12:25:04,577 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1/f/fdcfc2b4f61244028e163a86e4ae5481 for snapshot=snapshot_1696249503472_default_test-1696249492588 2023-10-02T12:25:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:25:04,990 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. 2023-10-02T12:25:04,990 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-10-02T12:25:04,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster(3995): Remote procedure done, pid=35 2023-10-02T12:25:04,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249503472_default_test-1696249492588 on region ff520da84fd90f4138d4152604b554a1 2023-10-02T12:25:04,993 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure ff520da84fd90f4138d4152604b554a1 2023-10-02T12:25:04,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-10-02T12:25:04,997 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249503472_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:25:04,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure ff520da84fd90f4138d4152604b554a1 in 1.0560 sec 2023-10-02T12:25:04,998 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249503472_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:25:04,999 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249503472_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:25:04,999 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249503472_default_test-1696249492588 2023-10-02T12:25:05,000 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_1696249503472_default_test-1696249492588 2023-10-02T12:25:05,420 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249503472_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:25:05,430 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249503472_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:25:05,431 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_1696249503472_default_test-1696249492588 to hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/snapshot_1696249503472_default_test-1696249492588 2023-10-02T12:25:05,433 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249503472_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:25:05,433 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249503472_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-10-02T12:25:05,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249503472_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } in 1.9410 sec 2023-10-02T12:25:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:25:05,608 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: default:test-1696249492588, procId: 34 completed 2023-10-02T12:25:05,609 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696249498461 2023-10-02T12:25:05,609 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:25:05,610 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-10-02T12:25:05,613 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249498461_test-1696249492588 2023-10-02T12:25:05,613 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249503472_default_test-1696249492588 to hdfs://localhost:36497/backupUT/backup_1696249498461/default/test-1696249492588/ 2023-10-02T12:25:05,625 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:25:05,668 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:36497, inputRoot=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a 2023-10-02T12:25:05,668 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43611,1696249478718_-876448392_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36497/backupUT/backup_1696249498461/default/test-1696249492588, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36497/backupUT/backup_1696249498461/default/test-1696249492588/.hbase-snapshot/.tmp/snapshot_1696249503472_default_test-1696249492588 2023-10-02T12:25:05,668 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:36497, inputRoot=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a, snapshotDir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/snapshot_1696249503472_default_test-1696249492588. 2023-10-02T12:25:05,704 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/snapshot_1696249503472_default_test-1696249492588 to hdfs://localhost:36497/backupUT/backup_1696249498461/default/test-1696249492588/.hbase-snapshot/.tmp/snapshot_1696249503472_default_test-1696249492588 2023-10-02T12:25:05,767 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:05,767 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:05,768 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:05,768 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:05,990 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:25:06,986 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/hadoop-2281593622073923250.jar 2023-10-02T12:25:06,986 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:06,987 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:06,988 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:06,988 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:06,989 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:06,990 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:06,990 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:06,991 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:06,992 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:06,993 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:25:06,994 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:25:06,995 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:25:06,995 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:25:06,996 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:25:06,997 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:25:06,997 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:25:06,998 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:25:06,999 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:25:06,999 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:25:07,000 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:25:07,001 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:25:07,007 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:25:07,007 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:25:07,008 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:25:07,008 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:25:07,009 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:25:07,010 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:25:07,010 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:25:07,096 WARN [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:25:07,128 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249503472_default_test-1696249492588' hfile list 2023-10-02T12:25:07,143 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-10-02T12:25:07,546 WARN [Listener at jenkins-hbase3.apache.org/37677 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/mapreduce.cluster.local.dir/1696249507420/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:25:07,546 WARN [Listener at jenkins-hbase3.apache.org/37677 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/mapreduce.cluster.local.dir/1696249507420/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:25:07,768 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:25:07,799 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1/f/fdcfc2b4f61244028e163a86e4ae5481 output=hdfs://localhost:36497/backupUT/backup_1696249498461/default/test-1696249492588/archive/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1/f/fdcfc2b4f61244028e163a86e4ae5481 2023-10-02T12:25:07,799 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-10-02T12:25:08,644 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:25:08,645 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:25:08,653 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249503472_default_test-1696249492588 2023-10-02T12:25:08,653 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249503472_default_test-1696249492588 finished. 2023-10-02T12:25:08,653 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1034): test-1696249492588 2023-10-02T12:25:08,729 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249498461 2023-10-02T12:25:08,729 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:25:09,147 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:36497/backupUT/backup_1696249498461/.backup.manifest 2023-10-02T12:25:09,147 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(358): Backup backup_1696249498461 finished: type=FULL,tablelist=test-1696249492588,targetRootDir=hdfs://localhost:36497/backupUT,startts=1696249503115,completets=1696249508726,bytescopied=0 2023-10-02T12:25:09,147 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:25:09,147 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249503472_default_test-1696249492588 2023-10-02T12:25:09,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249503472_default_test-1696249492588" 2023-10-02T12:25:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249503472_default_test-1696249492588 2023-10-02T12:25:09,168 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249503472_default_test-1696249492588 for backup backup_1696249498461 succeeded. 2023-10-02T12:25:09,170 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:25:09,186 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:25:09,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:25:09,190 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:25:09,200 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:25:09,219 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(376): Backup backup_1696249498461 completed. 2023-10-02T12:25:09,220 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:25:09,220 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x70409664 to 127.0.0.1:58172 2023-10-02T12:25:09,220 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:09,242 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] backup.TestBackupDeleteWithFailures(147): backup complete 2023-10-02T12:25:09,273 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:25:09,285 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:25:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249509285 (current time:1696249509285). 2023-10-02T12:25:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:25:09,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:25:09,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x70981a6f to 127.0.0.1:58172 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:25:09,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63d46aae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:25:09,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:25:09,296 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:34412, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:25:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x70981a6f to 127.0.0.1:58172 2023-10-02T12:25:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:09,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:25:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:25:09,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-10-02T12:25:09,304 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:25:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:25:09,305 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:25:09,309 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot, current working dir path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:25:09,313 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:25:09,347 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:25:09,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30}] 2023-10-02T12:25:09,349 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:09,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:25:09,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35033 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2023-10-02T12:25:09,501 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:09,502 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2819): Flushing 3dfc58254ae9ba5ba94c6dcd551aba30 2/2 column families, dataSize=1.14 KB heapSize=2.37 KB 2023-10-02T12:25:09,518 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/44756fad2d4c4c749f6666e97cd9aa7a is 169, key is trslm:hdfs://localhost:36497/backupUT\x00test-1696249492588/meta:log-roll-map/1696249508715/Put/seqid=0 2023-10-02T12:25:09,529 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=586 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/44756fad2d4c4c749f6666e97cd9aa7a 2023-10-02T12:25:09,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/3ba88e6f8052440dbf83bfb0e7a50568 is 310, key is session:backup_1696249498461/session:context/1696249509199/Put/seqid=0 2023-10-02T12:25:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:25:09,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:25:09,957 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=579 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/3ba88e6f8052440dbf83bfb0e7a50568 2023-10-02T12:25:09,967 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/44756fad2d4c4c749f6666e97cd9aa7a as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/44756fad2d4c4c749f6666e97cd9aa7a 2023-10-02T12:25:09,974 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/44756fad2d4c4c749f6666e97cd9aa7a, entries=5, sequenceid=18, filesize=5.5 K 2023-10-02T12:25:09,976 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/3ba88e6f8052440dbf83bfb0e7a50568 as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/3ba88e6f8052440dbf83bfb0e7a50568 2023-10-02T12:25:09,983 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/3ba88e6f8052440dbf83bfb0e7a50568, entries=2, sequenceid=18, filesize=5.3 K 2023-10-02T12:25:09,984 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1165, heapSize ~2.34 KB/2392, currentSize=0 B/0 for 3dfc58254ae9ba5ba94c6dcd551aba30 in 483ms, sequenceid=18, compaction requested=false 2023-10-02T12:25:09,984 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2520): Flush status journal for 3dfc58254ae9ba5ba94c6dcd551aba30: 2023-10-02T12:25:09,984 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. for snapshot_backup_system completed. 2023-10-02T12:25:09,985 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:25:09,985 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:25:09,985 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/44756fad2d4c4c749f6666e97cd9aa7a] hfiles 2023-10-02T12:25:09,985 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/44756fad2d4c4c749f6666e97cd9aa7a for snapshot=snapshot_backup_system 2023-10-02T12:25:09,985 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/9f3d5d24e61b47dca538b66fb59a927a, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/3ba88e6f8052440dbf83bfb0e7a50568] hfiles 2023-10-02T12:25:09,985 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/9f3d5d24e61b47dca538b66fb59a927a for snapshot=snapshot_backup_system 2023-10-02T12:25:09,985 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/3ba88e6f8052440dbf83bfb0e7a50568 for snapshot=snapshot_backup_system 2023-10-02T12:25:10,051 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-10-02T12:25:10,051 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-10-02T12:25:10,052 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-10-02T12:25:10,052 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-10-02T12:25:10,399 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:10,399 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=37 2023-10-02T12:25:10,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster(3995): Remote procedure done, pid=37 2023-10-02T12:25:10,400 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:10,401 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:10,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=37, resume processing ppid=36 2023-10-02T12:25:10,405 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, ppid=36, state=SUCCESS; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30 in 1.0550 sec 2023-10-02T12:25:10,405 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:25:10,406 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:25:10,406 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:25:10,406 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:25:10,408 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:25:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:25:10,422 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:25:10,432 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:25:10,432 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:25:10,434 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:25:10,434 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-10-02T12:25:10,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.1360 sec 2023-10-02T12:25:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:25:11,415 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 36 completed 2023-10-02T12:25:11,424 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupAdminImpl(227): Deleting backup backup_1696249498461 ... 2023-10-02T12:25:11,425 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-10-02T12:25:11,426 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] util.BackupUtils(435): Delete log files: default 2023-10-02T12:25:11,427 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1696249498461 2023-10-02T12:25:11,428 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:36497/backupUT/backup_1696249498461/default/test-1696249492588. 2023-10-02T12:25:11,429 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] util.BackupUtils(465): hdfs://localhost:36497/backupUT/backup_1696249498461/default is empty, remove it. 2023-10-02T12:25:11,440 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-10-02T12:25:11,440 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-10-02T12:25:11,446 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupAdminImpl(276): Delete backup backup_1696249498461 completed. 2023-10-02T12:25:11,455 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1034): test-1696249492588 2023-10-02T12:25:11,459 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:25:11,467 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:25:11,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:377) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:762) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-10-02T12:25:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.CallRunner(138): callId: 141 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:35550 deadline: 1696249571467, exception=java.io.IOException: preDeleteSnapshot 2023-10-02T12:25:11,594 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:25:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:377) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:762) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-10-02T12:25:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.CallRunner(138): callId: 143 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:35550 deadline: 1696249571594, exception=java.io.IOException: preDeleteSnapshot 2023-10-02T12:25:11,606 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-10-02T12:25:11,613 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin$18(967): Started disable of backup:system 2023-10-02T12:25:11,618 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable backup:system 2023-10-02T12:25:11,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=38, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-10-02T12:25:11,632 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249511632"}]},"ts":"1696249511632"} 2023-10-02T12:25:11,634 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-10-02T12:25:11,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:25:11,637 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-10-02T12:25:11,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=3dfc58254ae9ba5ba94c6dcd551aba30, UNASSIGN}] 2023-10-02T12:25:11,642 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=3dfc58254ae9ba5ba94c6dcd551aba30, UNASSIGN 2023-10-02T12:25:11,643 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=3dfc58254ae9ba5ba94c6dcd551aba30, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:11,646 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE; CloseRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30, server=jenkins-hbase3.apache.org,35033,1696249482294}] 2023-10-02T12:25:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:25:11,809 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(119): Close 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:11,809 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1668): Closing 3dfc58254ae9ba5ba94c6dcd551aba30, disabling compactions & flushes 2023-10-02T12:25:11,809 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1690): Closing region backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:11,809 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:11,809 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. after waiting 0 ms 2023-10-02T12:25:11,809 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:11,809 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(2819): Flushing 3dfc58254ae9ba5ba94c6dcd551aba30 2/2 column families, dataSize=179 B heapSize=912 B 2023-10-02T12:25:11,816 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/faffafb4cc504681a6a9e1044cb672ef is 91, key is incrbackupset:hdfs://localhost:36497/backupUT/meta:test-1696249492588/1696249511456/Put/seqid=0 2023-10-02T12:25:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:25:12,227 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=124 B at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/faffafb4cc504681a6a9e1044cb672ef 2023-10-02T12:25:12,237 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for faffafb4cc504681a6a9e1044cb672ef 2023-10-02T12:25:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:25:12,242 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/f387dd0b34d54c129401f2a2a05788b1 is 59, key is session:backup_1696249498461/session:/1696249511444/DeleteFamily/seqid=0 2023-10-02T12:25:12,657 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55 B at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/f387dd0b34d54c129401f2a2a05788b1 2023-10-02T12:25:12,666 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f387dd0b34d54c129401f2a2a05788b1 2023-10-02T12:25:12,667 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/faffafb4cc504681a6a9e1044cb672ef as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/faffafb4cc504681a6a9e1044cb672ef 2023-10-02T12:25:12,675 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for faffafb4cc504681a6a9e1044cb672ef 2023-10-02T12:25:12,675 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/faffafb4cc504681a6a9e1044cb672ef, entries=2, sequenceid=24, filesize=5.2 K 2023-10-02T12:25:12,676 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/f387dd0b34d54c129401f2a2a05788b1 as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/f387dd0b34d54c129401f2a2a05788b1 2023-10-02T12:25:12,684 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f387dd0b34d54c129401f2a2a05788b1 2023-10-02T12:25:12,684 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/f387dd0b34d54c129401f2a2a05788b1, entries=1, sequenceid=24, filesize=5.1 K 2023-10-02T12:25:12,686 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(3022): Finished flush of dataSize ~179 B/179, heapSize ~880 B/880, currentSize=0 B/0 for 3dfc58254ae9ba5ba94c6dcd551aba30 in 877ms, sequenceid=24, compaction requested=true 2023-10-02T12:25:12,703 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/recovered.edits/27.seqid, newMaxSeqId=27, maxSeqId=1 2023-10-02T12:25:12,704 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:25:12,705 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1909): Closed backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:12,705 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1622): Region close journal for 3dfc58254ae9ba5ba94c6dcd551aba30: 2023-10-02T12:25:12,707 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(163): Closed 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:12,708 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=3dfc58254ae9ba5ba94c6dcd551aba30, regionState=CLOSED 2023-10-02T12:25:12,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=40, resume processing ppid=39 2023-10-02T12:25:12,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, ppid=39, state=SUCCESS; CloseRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30, server=jenkins-hbase3.apache.org,35033,1696249482294 in 1.0640 sec 2023-10-02T12:25:12,714 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=39, resume processing ppid=38 2023-10-02T12:25:12,714 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, ppid=38, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=3dfc58254ae9ba5ba94c6dcd551aba30, UNASSIGN in 1.0720 sec 2023-10-02T12:25:12,715 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249512715"}]},"ts":"1696249512715"} 2023-10-02T12:25:12,716 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-10-02T12:25:12,718 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-10-02T12:25:12,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, state=SUCCESS; DisableTableProcedure table=backup:system in 1.0990 sec 2023-10-02T12:25:12,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:25:12,741 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: backup:system, procId: 38 completed 2023-10-02T12:25:12,751 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin(2732): Taking restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1696249512751 2023-10-02T12:25:12,752 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1696249512751 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:25:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249512753 (current time:1696249512753). 2023-10-02T12:25:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:25:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1696249512751 VERSION not specified, setting to 2 2023-10-02T12:25:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3efbfa1d to 127.0.0.1:58172 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:25:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bb404b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:25:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:25:12,764 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50078, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:25:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3efbfa1d to 127.0.0.1:58172 2023-10-02T12:25:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:12,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:25:12,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=41, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249512751 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:25:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249512751 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 41 2023-10-02T12:25:12,774 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249512751 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:25:12,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-10-02T12:25:12,776 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249512751 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:25:12,779 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot, current working dir path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696249512751 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:25:12,782 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249512751 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:25:12,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-10-02T12:25:13,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-10-02T12:25:13,190 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249512751 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-10-02T12:25:13,193 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-10-02T12:25:13,193 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-10-02T12:25:13,193 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:25:13,197 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/44756fad2d4c4c749f6666e97cd9aa7a, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/faffafb4cc504681a6a9e1044cb672ef] hfiles 2023-10-02T12:25:13,197 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/2): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/44756fad2d4c4c749f6666e97cd9aa7a 2023-10-02T12:25:13,197 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/2): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/faffafb4cc504681a6a9e1044cb672ef 2023-10-02T12:25:13,197 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:25:13,203 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/3ba88e6f8052440dbf83bfb0e7a50568, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/9f3d5d24e61b47dca538b66fb59a927a, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/f387dd0b34d54c129401f2a2a05788b1] hfiles 2023-10-02T12:25:13,203 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/3ba88e6f8052440dbf83bfb0e7a50568 2023-10-02T12:25:13,203 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/9f3d5d24e61b47dca538b66fb59a927a 2023-10-02T12:25:13,204 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/f387dd0b34d54c129401f2a2a05788b1 2023-10-02T12:25:13,219 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 3dfc58254ae9ba5ba94c6dcd551aba30, NAME => 'backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-10-02T12:25:13,221 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249512751 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:25:13,221 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249512751 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:25:13,222 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1696249512751 2023-10-02T12:25:13,222 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696249512751 2023-10-02T12:25:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-10-02T12:25:13,637 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249512751 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:25:13,648 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249512751 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:25:13,649 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696249512751 to hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1696249512751 2023-10-02T12:25:13,651 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=41, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249512751 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:25:13,651 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249512751 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 41 2023-10-02T12:25:13,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=41, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249512751 table=backup:system type=FLUSH ttl=0 } in 884 msec 2023-10-02T12:25:13,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-10-02T12:25:13,883 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 41 completed 2023-10-02T12:25:13,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=42, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249509285 type: FLUSH version: 2 ttl: 0 ) 2023-10-02T12:25:13,915 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-10-02T12:25:13,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:25:13,946 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/.tabledesc/.tableinfo.0000000001.635 2023-10-02T12:25:13,947 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-10-02T12:25:13,948 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:25:13,955 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249509285 type: FLUSH version: 2 ttl: 0 2023-10-02T12:25:13,956 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system 2023-10-02T12:25:13,960 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-10-02T12:25:13,960 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:13,978 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/faffafb4cc504681a6a9e1044cb672ef to hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/archive/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/faffafb4cc504681a6a9e1044cb672ef 2023-10-02T12:25:13,978 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:25:13,983 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/f387dd0b34d54c129401f2a2a05788b1 to hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/archive/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/f387dd0b34d54c129401f2a2a05788b1 2023-10-02T12:25:13,983 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:25:13,983 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249509285 type: FLUSH version: 2 ttl: 0 2023-10-02T12:25:13,985 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696249513985"}]},"ts":"9223372036854775807"} 2023-10-02T12:25:13,990 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:25:13,990 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 3dfc58254ae9ba5ba94c6dcd551aba30, NAME => 'backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:25:13,990 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1696249513986"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249513986"}]},"ts":"1696249513986"} 2023-10-02T12:25:13,992 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:25:13,992 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-10-02T12:25:13,992 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 3dfc58254ae9ba5ba94c6dcd551aba30, NAME => 'backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:25:13,993 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-10-02T12:25:13,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249509285 type: FLUSH version: 2 ttl: 0 ) in 92 msec 2023-10-02T12:25:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=42 2023-10-02T12:25:14,022 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin$TableFuture(3677): Operation: MODIFY, Table Name: backup:system, procId: 42 completed 2023-10-02T12:25:14,022 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin(2765): Deleting restore-failsafe snapshot: hbase-failsafe-snapshot_backup_system-1696249512751 2023-10-02T12:25:14,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1696249512751" 2023-10-02T12:25:14,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:377) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:762) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-10-02T12:25:14,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.CallRunner(138): callId: 184 service: MasterService methodName: DeleteSnapshot size: 86 connection: 172.31.12.81:35550 deadline: 1696249574023, exception=java.io.IOException: preDeleteSnapshot 2023-10-02T12:25:14,129 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1696249512751" 2023-10-02T12:25:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:377) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:762) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-10-02T12:25:14,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.CallRunner(138): callId: 186 service: MasterService methodName: DeleteSnapshot size: 86 connection: 172.31.12.81:35550 deadline: 1696249574128, exception=java.io.IOException: preDeleteSnapshot 2023-10-02T12:25:14,133 ERROR [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin(2768): Unable to remove the failsafe snapshot: hbase-failsafe-snapshot_backup_system-1696249512751 org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=2, exceptions: 2023-10-02T12:25:14.027Z, RpcRetryingCaller{globalStartTime=2023-10-02T12:25:14.022Z, pause=100, maxAttempts=2}, java.io.IOException: java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:377) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:762) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2023-10-02T12:25:14.133Z, RpcRetryingCaller{globalStartTime=2023-10-02T12:25:14.022Z, pause=100, maxAttempts=2}, java.io.IOException: java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:377) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:762) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:142) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3105) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3097) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.deleteSnapshot(HBaseAdmin.java:3005) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.restoreSnapshot(HBaseAdmin.java:2766) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.restoreSnapshot(HBaseAdmin.java:2704) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.restoreSnapshot(HBaseAdmin.java:2669) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.backup.impl.BackupSystemTable.restoreFromSnapshot(BackupSystemTable.java:1543) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.deleteBackups(BackupAdminImpl.java:155) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures.testBackupDeleteWithFailuresAfter(TestBackupDeleteWithFailures.java:161) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures.testBackupDeleteWithFailures(TestBackupDeleteWithFailures.java:136) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: java.io.IOException: java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:377) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:762) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.MasterCallable.call(MasterCallable.java:101) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] ... 34 more Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:377) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:762) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-10-02T12:25:14,137 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin$17(907): Started enable of backup:system 2023-10-02T12:25:14,143 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster$11(2631): Client=jenkins//172.31.12.81 enable backup:system 2023-10-02T12:25:14,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=43, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-10-02T12:25:14,158 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-10-02T12:25:14,159 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249514158"}]},"ts":"1696249514158"} 2023-10-02T12:25:14,160 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-10-02T12:25:14,163 DEBUG [PEWorker-2 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-10-02T12:25:14,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=3dfc58254ae9ba5ba94c6dcd551aba30, ASSIGN}] 2023-10-02T12:25:14,166 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=3dfc58254ae9ba5ba94c6dcd551aba30, ASSIGN 2023-10-02T12:25:14,167 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=44, ppid=43, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=3dfc58254ae9ba5ba94c6dcd551aba30, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-10-02T12:25:14,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-10-02T12:25:14,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-10-02T12:25:14,318 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=44 updating hbase:meta row=3dfc58254ae9ba5ba94c6dcd551aba30, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:14,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE; OpenRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30, server=jenkins-hbase3.apache.org,35033,1696249482294}] 2023-10-02T12:25:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-10-02T12:25:14,478 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(130): Open backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:14,478 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7254): Opening region: {ENCODED => 3dfc58254ae9ba5ba94c6dcd551aba30, NAME => 'backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:25:14,479 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:25:14,479 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:14,479 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(888): Instantiated backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:25:14,479 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7296): checking encryption for 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:14,479 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7299): checking classloading for 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:14,483 INFO [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:14,485 INFO [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3dfc58254ae9ba5ba94c6dcd551aba30 columnFamilyName meta 2023-10-02T12:25:14,485 DEBUG [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:25:14,496 DEBUG [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/44756fad2d4c4c749f6666e97cd9aa7a 2023-10-02T12:25:14,497 INFO [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] regionserver.HStore(324): Store=3dfc58254ae9ba5ba94c6dcd551aba30/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:25:14,497 INFO [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:14,498 INFO [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3dfc58254ae9ba5ba94c6dcd551aba30 columnFamilyName session 2023-10-02T12:25:14,498 DEBUG [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:25:14,518 DEBUG [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/3ba88e6f8052440dbf83bfb0e7a50568 2023-10-02T12:25:14,526 DEBUG [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/9f3d5d24e61b47dca538b66fb59a927a 2023-10-02T12:25:14,526 INFO [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] regionserver.HStore(324): Store=3dfc58254ae9ba5ba94c6dcd551aba30/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:25:14,527 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:14,529 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:14,531 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:25:14,532 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1076): writing seq id for 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:14,534 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): Opened 3dfc58254ae9ba5ba94c6dcd551aba30; next sequenceid=28; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9926663680, jitterRate=-0.07550740242004395}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:25:14,535 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(992): Region open journal for 3dfc58254ae9ba5ba94c6dcd551aba30: 2023-10-02T12:25:14,536 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30., pid=45, masterSystemTime=1696249514473 2023-10-02T12:25:14,538 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:14,538 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(158): Opened backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:14,538 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=44 updating hbase:meta row=3dfc58254ae9ba5ba94c6dcd551aba30, regionState=OPEN, openSeqNum=28, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:14,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=45, resume processing ppid=44 2023-10-02T12:25:14,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, ppid=44, state=SUCCESS; OpenRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30, server=jenkins-hbase3.apache.org,35033,1696249482294 in 220 msec 2023-10-02T12:25:14,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=44, resume processing ppid=43 2023-10-02T12:25:14,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, ppid=43, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=3dfc58254ae9ba5ba94c6dcd551aba30, ASSIGN in 380 msec 2023-10-02T12:25:14,546 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249514546"}]},"ts":"1696249514546"} 2023-10-02T12:25:14,548 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-10-02T12:25:14,550 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-10-02T12:25:14,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, state=SUCCESS; EnableTableProcedure table=backup:system in 406 msec 2023-10-02T12:25:14,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=43 2023-10-02T12:25:14,773 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] client.HBaseAdmin$TableFuture(3677): Operation: ENABLE, Table Name: backup:system, procId: 43 completed 2023-10-02T12:25:14,773 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-10-02T12:25:14,773 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:25:14,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:25:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:377) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:762) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-10-02T12:25:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.CallRunner(138): callId: 200 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:35550 deadline: 1696249574784, exception=java.io.IOException: preDeleteSnapshot 2023-10-02T12:25:14,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:25:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: preDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preDeleteSnapshot(TestBackupDeleteWithFailures.java:103) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1155) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$88.call(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preDeleteSnapshot(MasterCoprocessorHost.java:1152) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:377) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:762) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-10-02T12:25:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.CallRunner(138): callId: 202 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:35550 deadline: 1696249574891, exception=java.io.IOException: preDeleteSnapshot 2023-10-02T12:25:14,896 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:25:14,933 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6654b9fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:25:14,941 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b62457, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:25:14,942 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:25:14,945 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46194, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:25:14,953 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:25:14,956 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50090, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService REPAIR status: no failed sessions found. Checking failed delete backup operation ... Found failed DELETE operation for: backup_1696249498461 Running DELETE again ... 2023-10-02T12:25:14,983 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-10-02T12:25:14,997 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable backup:system 2023-10-02T12:25:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=46, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=backup:system 2023-10-02T12:25:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-10-02T12:25:15,004 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249515004"}]},"ts":"1696249515004"} 2023-10-02T12:25:15,006 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLING in hbase:meta 2023-10-02T12:25:15,007 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set backup:system to state=DISABLING 2023-10-02T12:25:15,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=3dfc58254ae9ba5ba94c6dcd551aba30, UNASSIGN}] 2023-10-02T12:25:15,010 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=47, ppid=46, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=backup:system, region=3dfc58254ae9ba5ba94c6dcd551aba30, UNASSIGN 2023-10-02T12:25:15,011 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=47 updating hbase:meta row=3dfc58254ae9ba5ba94c6dcd551aba30, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:15,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE; CloseRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30, server=jenkins-hbase3.apache.org,35033,1696249482294}] 2023-10-02T12:25:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-10-02T12:25:15,145 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-10-02T12:25:15,147 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46202, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-10-02T12:25:15,165 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] handler.UnassignRegionHandler(119): Close 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:15,167 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1668): Closing 3dfc58254ae9ba5ba94c6dcd551aba30, disabling compactions & flushes 2023-10-02T12:25:15,167 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1690): Closing region backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:15,167 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:15,167 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. after waiting 0 ms 2023-10-02T12:25:15,167 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:15,167 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(2819): Flushing 3dfc58254ae9ba5ba94c6dcd551aba30 2/2 column families, dataSize=44 B heapSize=632 B 2023-10-02T12:25:15,189 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/03627f5f3e6a44988927a822190fd399 is 48, key is activesession:/session:c/1696249514898/Put/seqid=0 2023-10-02T12:25:15,199 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44 B at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/03627f5f3e6a44988927a822190fd399 2023-10-02T12:25:15,209 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/03627f5f3e6a44988927a822190fd399 as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/03627f5f3e6a44988927a822190fd399 2023-10-02T12:25:15,217 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/03627f5f3e6a44988927a822190fd399, entries=1, sequenceid=31, filesize=4.9 K 2023-10-02T12:25:15,219 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(3022): Finished flush of dataSize ~44 B/44, heapSize ~360 B/360, currentSize=0 B/0 for 3dfc58254ae9ba5ba94c6dcd551aba30 in 51ms, sequenceid=31, compaction requested=true 2023-10-02T12:25:15,219 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-10-02T12:25:15,238 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/recovered.edits/34.seqid, newMaxSeqId=34, maxSeqId=27 2023-10-02T12:25:15,239 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:25:15,239 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1909): Closed backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:15,239 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] regionserver.HRegion(1622): Region close journal for 3dfc58254ae9ba5ba94c6dcd551aba30: 2023-10-02T12:25:15,242 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=48}] handler.UnassignRegionHandler(163): Closed 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:15,247 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=47 updating hbase:meta row=3dfc58254ae9ba5ba94c6dcd551aba30, regionState=CLOSED 2023-10-02T12:25:15,252 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=48, resume processing ppid=47 2023-10-02T12:25:15,252 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, ppid=47, state=SUCCESS; CloseRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30, server=jenkins-hbase3.apache.org,35033,1696249482294 in 236 msec 2023-10-02T12:25:15,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=47, resume processing ppid=46 2023-10-02T12:25:15,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, ppid=46, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=3dfc58254ae9ba5ba94c6dcd551aba30, UNASSIGN in 244 msec 2023-10-02T12:25:15,258 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249515258"}]},"ts":"1696249515258"} 2023-10-02T12:25:15,260 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=DISABLED in hbase:meta 2023-10-02T12:25:15,262 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set backup:system to state=DISABLED 2023-10-02T12:25:15,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, state=SUCCESS; DisableTableProcedure table=backup:system in 265 msec 2023-10-02T12:25:15,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=46 2023-10-02T12:25:15,328 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=hbase-failsafe-snapshot_backup_system-1696249515327 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:25:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249515328 (current time:1696249515328). 2023-10-02T12:25:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:25:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot hbase-failsafe-snapshot_backup_system-1696249515327 VERSION not specified, setting to 2 2023-10-02T12:25:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b55d2be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:25:15,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:25:15,348 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50106, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:25:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:25:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=49, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249515327 table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:25:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249515327 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 49 2023-10-02T12:25:15,354 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249515327 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:25:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-10-02T12:25:15,355 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249515327 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:25:15,359 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot, current working dir path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696249515327 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:25:15,363 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249515327 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:25:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-10-02T12:25:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-10-02T12:25:15,785 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_SNAPSHOT_CLOSED_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249515327 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_CLOSED_REGIONS 2023-10-02T12:25:15,787 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(299): Storing region-info for snapshot. 2023-10-02T12:25:15,787 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(304): Creating references for hfiles 2023-10-02T12:25:15,787 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:25:15,788 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/44756fad2d4c4c749f6666e97cd9aa7a] hfiles 2023-10-02T12:25:15,788 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/1): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/44756fad2d4c4c749f6666e97cd9aa7a 2023-10-02T12:25:15,789 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:25:15,791 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(360): Adding snapshot references for [hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/03627f5f3e6a44988927a822190fd399, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/3ba88e6f8052440dbf83bfb0e7a50568, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/9f3d5d24e61b47dca538b66fb59a927a] hfiles 2023-10-02T12:25:15,791 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (1/3): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/03627f5f3e6a44988927a822190fd399 2023-10-02T12:25:15,791 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (2/3): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/3ba88e6f8052440dbf83bfb0e7a50568 2023-10-02T12:25:15,791 DEBUG [ClosedRegionsSnapshotPool-pool-0 {}] snapshot.SnapshotManifest(368): Adding reference for hfile (3/3): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/9f3d5d24e61b47dca538b66fb59a927a 2023-10-02T12:25:15,799 INFO [ClosedRegionsSnapshotPool-pool-0 {}] procedure.SnapshotProcedure$1(378): take snapshot region={ENCODED => 3dfc58254ae9ba5ba94c6dcd551aba30, NAME => 'backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.', STARTKEY => '', ENDKEY => ''}, table=backup:system 2023-10-02T12:25:15,800 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249515327 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:25:15,801 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249515327 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:25:15,801 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for hbase-failsafe-snapshot_backup_system-1696249515327 2023-10-02T12:25:15,802 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696249515327 2023-10-02T12:25:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-10-02T12:25:16,217 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249515327 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:25:16,227 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249515327 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:25:16,227 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/hbase-failsafe-snapshot_backup_system-1696249515327 to hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/hbase-failsafe-snapshot_backup_system-1696249515327 2023-10-02T12:25:16,229 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=49, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249515327 table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:25:16,229 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249515327 table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 49 2023-10-02T12:25:16,230 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=49, snapshot={ ss=hbase-failsafe-snapshot_backup_system-1696249515327 table=backup:system type=FLUSH ttl=0 } in 879 msec 2023-10-02T12:25:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=49 2023-10-02T12:25:16,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=50, state=RUNNABLE:RESTORE_SNAPSHOT_PRE_OPERATION; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249509285 type: FLUSH version: 2 ttl: 0 ) 2023-10-02T12:25:16,473 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(1051): Restore snapshot=snapshot_backup_system as table=backup:system 2023-10-02T12:25:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:25:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:25:16,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:25:16,897 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/.tabledesc/.tableinfo.0000000002.635 2023-10-02T12:25:16,897 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/.tabledesc/.tableinfo.0000000003.635 2023-10-02T12:25:16,899 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(386): Starting restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:25:16,902 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(181): starting restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249509285 type: FLUSH version: 2 ttl: 0 2023-10-02T12:25:16,902 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(846): get table regions: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system 2023-10-02T12:25:16,905 DEBUG [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(858): found 1 regions for table=backup:system 2023-10-02T12:25:16,905 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(227): region to restore: 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:16,910 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:25:16,914 DEBUG [RestoreSnapshot-pool-0 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/03627f5f3e6a44988927a822190fd399 to hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/archive/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/03627f5f3e6a44988927a822190fd399 2023-10-02T12:25:16,914 DEBUG [RestoreSnapshot-pool-0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:25:16,914 INFO [PEWorker-4 {}] snapshot.RestoreSnapshotHelper(270): finishing restore table regions using snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249509285 type: FLUSH version: 2 ttl: 0 2023-10-02T12:25:16,915 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696249516915"}]},"ts":"9223372036854775807"} 2023-10-02T12:25:16,918 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:25:16,918 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 3dfc58254ae9ba5ba94c6dcd551aba30, NAME => 'backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:25:16,918 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1696249516916"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249516916"}]},"ts":"1696249516916"} 2023-10-02T12:25:16,919 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:25:16,920 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1830): Overwritten 1 regions to Meta 2023-10-02T12:25:16,920 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1831): Overwritten regions: [{ENCODED => 3dfc58254ae9ba5ba94c6dcd551aba30, NAME => 'backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:25:16,920 INFO [PEWorker-4 {}] procedure.RestoreSnapshotProcedure(457): Restore snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } on table=backup:system completed! 2023-10-02T12:25:16,923 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, state=SUCCESS; RestoreSnapshotProcedure (table=backup:system snapshot=name: "snapshot_backup_system" table: "backup:system" creation_time: 1696249509285 type: FLUSH version: 2 ttl: 0 ) in 449 msec 2023-10-02T12:25:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:25:17,081 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1696249515327" 2023-10-02T12:25:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1696249515327 2023-10-02T12:25:17,086 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster$11(2631): Client=jenkins//172.31.12.81 enable backup:system 2023-10-02T12:25:17,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=51, state=RUNNABLE:ENABLE_TABLE_PREPARE; EnableTableProcedure table=backup:system 2023-10-02T12:25:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=51 2023-10-02T12:25:17,090 INFO [PEWorker-2 {}] procedure.EnableTableProcedure(296): Attempting to enable the table backup:system 2023-10-02T12:25:17,090 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249517090"}]},"ts":"1696249517090"} 2023-10-02T12:25:17,092 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-10-02T12:25:17,094 DEBUG [PEWorker-2 {}] procedure.EnableTableProcedure(107): No change in number of region replicas (configuredReplicaCount=1); assigning. 2023-10-02T12:25:17,094 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=3dfc58254ae9ba5ba94c6dcd551aba30, ASSIGN}] 2023-10-02T12:25:17,096 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=3dfc58254ae9ba5ba94c6dcd551aba30, ASSIGN 2023-10-02T12:25:17,097 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=3dfc58254ae9ba5ba94c6dcd551aba30, ASSIGN; state=CLOSED, location=null; forceNewPlan=false, retain=false 2023-10-02T12:25:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=51 2023-10-02T12:25:17,248 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=3dfc58254ae9ba5ba94c6dcd551aba30, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:17,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; OpenRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30, server=jenkins-hbase3.apache.org,35033,1696249482294}] 2023-10-02T12:25:17,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=51 2023-10-02T12:25:17,406 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(130): Open backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:17,406 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7254): Opening region: {ENCODED => 3dfc58254ae9ba5ba94c6dcd551aba30, NAME => 'backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:25:17,406 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:25:17,407 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:17,407 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(888): Instantiated backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:25:17,407 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7296): checking encryption for 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:17,407 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7299): checking classloading for 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:17,409 INFO [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:17,410 INFO [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3dfc58254ae9ba5ba94c6dcd551aba30 columnFamilyName meta 2023-10-02T12:25:17,410 DEBUG [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:25:17,417 DEBUG [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/44756fad2d4c4c749f6666e97cd9aa7a 2023-10-02T12:25:17,417 INFO [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] regionserver.HStore(324): Store=3dfc58254ae9ba5ba94c6dcd551aba30/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:25:17,417 INFO [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:17,418 INFO [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3dfc58254ae9ba5ba94c6dcd551aba30 columnFamilyName session 2023-10-02T12:25:17,418 DEBUG [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:25:17,425 DEBUG [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/3ba88e6f8052440dbf83bfb0e7a50568 2023-10-02T12:25:17,431 DEBUG [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/9f3d5d24e61b47dca538b66fb59a927a 2023-10-02T12:25:17,432 INFO [StoreOpener-3dfc58254ae9ba5ba94c6dcd551aba30-1 {}] regionserver.HStore(324): Store=3dfc58254ae9ba5ba94c6dcd551aba30/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:25:17,433 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:17,438 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:17,443 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:25:17,445 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1076): writing seq id for 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:17,446 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): Opened 3dfc58254ae9ba5ba94c6dcd551aba30; next sequenceid=35; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12069216320, jitterRate=0.1240333616733551}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:25:17,446 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(992): Region open journal for 3dfc58254ae9ba5ba94c6dcd551aba30: 2023-10-02T12:25:17,447 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30., pid=53, masterSystemTime=1696249517402 2023-10-02T12:25:17,455 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:17,455 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(158): Opened backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:17,455 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=52 updating hbase:meta row=3dfc58254ae9ba5ba94c6dcd551aba30, regionState=OPEN, openSeqNum=35, regionLocation=jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:17,460 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=53, resume processing ppid=52 2023-10-02T12:25:17,460 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, ppid=52, state=SUCCESS; OpenRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30, server=jenkins-hbase3.apache.org,35033,1696249482294 in 207 msec 2023-10-02T12:25:17,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=52, resume processing ppid=51 2023-10-02T12:25:17,462 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, ppid=51, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=3dfc58254ae9ba5ba94c6dcd551aba30, ASSIGN in 366 msec 2023-10-02T12:25:17,463 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249517463"}]},"ts":"1696249517463"} 2023-10-02T12:25:17,465 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-10-02T12:25:17,467 INFO [PEWorker-1 {}] procedure.EnableTableProcedure(310): Table 'backup:system' was successfully enabled. 2023-10-02T12:25:17,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, state=SUCCESS; EnableTableProcedure table=backup:system in 382 msec 2023-10-02T12:25:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=51 2023-10-02T12:25:17,694 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1545): Done restoring backup system table 2023-10-02T12:25:17,694 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:25:17,710 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:25:17,727 WARN [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupAdminImpl(128): Backup system table snapshot exists 2023-10-02T12:25:17,732 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupAdminImpl(227): Deleting backup backup_1696249498461 ... 2023-10-02T12:25:17,734 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1696249498461 2023-10-02T12:25:17,736 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:36497/backupUT/backup_1696249498461/default/test-1696249492588. 2023-10-02T12:25:17,737 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] util.BackupUtils(465): hdfs://localhost:36497/backupUT/backup_1696249498461/default is empty, remove it. 2023-10-02T12:25:17,745 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-10-02T12:25:17,746 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-10-02T12:25:17,747 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupAdminImpl(276): Delete backup backup_1696249498461 completed. 2023-10-02T12:25:17,752 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1034): test-1696249492588 2023-10-02T12:25:17,755 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:25:17,765 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:25:17,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:25:17,768 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:25:17,768 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(657): Finish backup exclusive operation DELETE operation finished OK: backup_1696249498461 No failed backup MERGE operation found 2023-10-02T12:25:17,772 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:25:17,776 ERROR [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1579): Snapshot snapshot_backup_system does not exists 2023-10-02T12:25:17,777 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:17,777 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:17,780 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] backup.TestBackupDeleteWithFailures(143): test repair backup delete on a single table with data and failures POST_DELETE_SNAPSHOT_FAILURE 2023-10-02T12:25:17,786 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@547ebd7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:25:17,790 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:25:17,792 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50120, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:25:17,794 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:25:17,797 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:46212, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:25:17,811 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:25:17,818 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:25:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249517818 (current time:1696249517818). 2023-10-02T12:25:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:25:17,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:25:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14575442, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:25:17,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:25:17,832 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50136, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:25:17,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:17,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:25:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=54, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:25:17,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 54 2023-10-02T12:25:17,837 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:25:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-10-02T12:25:17,838 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:25:17,841 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot, current working dir path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:25:17,844 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:25:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-10-02T12:25:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-10-02T12:25:18,254 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:25:18,255 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30}] 2023-10-02T12:25:18,256 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:18,264 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:25:18,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35033 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=55 2023-10-02T12:25:18,408 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:18,410 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2819): Flushing 3dfc58254ae9ba5ba94c6dcd551aba30 2/2 column families, dataSize=417 B heapSize=1.49 KB 2023-10-02T12:25:18,427 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/ca7ddbd11d4149f288b81612e951712f is 91, key is incrbackupset:hdfs://localhost:36497/backupUT/meta:test-1696249492588/1696249517753/Put/seqid=0 2023-10-02T12:25:18,433 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=184 B at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/ca7ddbd11d4149f288b81612e951712f 2023-10-02T12:25:18,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-10-02T12:25:18,444 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ca7ddbd11d4149f288b81612e951712f 2023-10-02T12:25:18,462 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/06206d275c734da89d8139ad46731eb7 is 59, key is session:backup_1696249498461/session:/1696249517746/DeleteFamily/seqid=0 2023-10-02T12:25:18,867 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=233 B at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/06206d275c734da89d8139ad46731eb7 2023-10-02T12:25:18,874 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 06206d275c734da89d8139ad46731eb7 2023-10-02T12:25:18,875 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/ca7ddbd11d4149f288b81612e951712f as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/ca7ddbd11d4149f288b81612e951712f 2023-10-02T12:25:18,881 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ca7ddbd11d4149f288b81612e951712f 2023-10-02T12:25:18,881 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/ca7ddbd11d4149f288b81612e951712f, entries=2, sequenceid=45, filesize=5.2 K 2023-10-02T12:25:18,882 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/06206d275c734da89d8139ad46731eb7 as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/06206d275c734da89d8139ad46731eb7 2023-10-02T12:25:18,888 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 06206d275c734da89d8139ad46731eb7 2023-10-02T12:25:18,889 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/06206d275c734da89d8139ad46731eb7, entries=2, sequenceid=45, filesize=5.2 K 2023-10-02T12:25:18,890 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(3022): Finished flush of dataSize ~417 B/417, heapSize ~1.46 KB/1496, currentSize=0 B/0 for 3dfc58254ae9ba5ba94c6dcd551aba30 in 480ms, sequenceid=45, compaction requested=true 2023-10-02T12:25:18,890 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.HRegion(2520): Flush status journal for 3dfc58254ae9ba5ba94c6dcd551aba30: 2023-10-02T12:25:18,890 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. for snapshot_backup_system completed. 2023-10-02T12:25:18,890 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:25:18,890 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:25:18,890 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/44756fad2d4c4c749f6666e97cd9aa7a, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/ca7ddbd11d4149f288b81612e951712f] hfiles 2023-10-02T12:25:18,890 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/44756fad2d4c4c749f6666e97cd9aa7a for snapshot=snapshot_backup_system 2023-10-02T12:25:18,890 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/ca7ddbd11d4149f288b81612e951712f for snapshot=snapshot_backup_system 2023-10-02T12:25:18,890 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/9f3d5d24e61b47dca538b66fb59a927a, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/3ba88e6f8052440dbf83bfb0e7a50568, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/06206d275c734da89d8139ad46731eb7] hfiles 2023-10-02T12:25:18,890 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/9f3d5d24e61b47dca538b66fb59a927a for snapshot=snapshot_backup_system 2023-10-02T12:25:18,890 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/3ba88e6f8052440dbf83bfb0e7a50568 for snapshot=snapshot_backup_system 2023-10-02T12:25:18,890 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/06206d275c734da89d8139ad46731eb7 for snapshot=snapshot_backup_system 2023-10-02T12:25:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-10-02T12:25:19,299 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:19,299 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=55 2023-10-02T12:25:19,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster(3995): Remote procedure done, pid=55 2023-10-02T12:25:19,300 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:19,301 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:19,304 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=55, resume processing ppid=54 2023-10-02T12:25:19,304 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:25:19,304 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=55, ppid=54, state=SUCCESS; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30 in 1.0480 sec 2023-10-02T12:25:19,305 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:25:19,306 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:25:19,306 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:25:19,306 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:25:19,724 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:25:19,736 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:25:19,736 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:25:19,738 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=54, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:25:19,738 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 54 2023-10-02T12:25:19,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=54, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=54, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.9040 sec 2023-10-02T12:25:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-10-02T12:25:19,945 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(115): Backup backup_1696249517787 started at 1696249519945. 2023-10-02T12:25:19,947 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(119): Backup session backup_1696249517787 has been started. 2023-10-02T12:25:19,948 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:25:19,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:25:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:25:19,949 INFO [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:25:19,949 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:25:19,951 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:25:19,951 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:25:19,951 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:25:19,953 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:19,953 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:25:19,953 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:25:19,954 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:25:19,954 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:25:19,954 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:25:19,954 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:25:19,954 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:25:19,955 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:25:19,955 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:25:19,955 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:25:19,955 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:25:19,956 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:25:19,956 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:25:19,956 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,35033,1696249482294' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:25:19,958 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:25:19,958 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:19,958 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:25:19,958 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:25:19,958 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:25:19,959 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:25:19,959 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:25:19,959 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:25:19,960 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:19,960 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:25:19,960 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,35033,1696249482294' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:25:19,960 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@75c9ce5e[Count = 0] remaining members to acquire global barrier 2023-10-02T12:25:19,960 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:25:19,960 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:25:19,968 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:25:19,968 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:25:19,968 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:25:19,968 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:25:19,968 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:25:19,968 DEBUG [rs(jenkins-hbase3.apache.org,35033,1696249482294)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:19,968 INFO [rs(jenkins-hbase3.apache.org,35033,1696249482294)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696249503214 highest: 1696249503214 on jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:19,968 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C35033%2C1696249482294:(num 1696249503214) roll requested 2023-10-02T12:25:19,980 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.1696249519968, exclude list is [], retry=0 2023-10-02T12:25:19,984 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38869,DS-d3b004e5-e24b-4408-a9bf-421ca1308ded,DISK] 2023-10-02T12:25:19,986 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.1696249503214 with entries=35, filesize=7.33 KB; new WAL /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.1696249519968 2023-10-02T12:25:19,986 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38869,DS-d3b004e5-e24b-4408-a9bf-421ca1308ded,DISK]] 2023-10-02T12:25:19,986 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.1696249503214 is not closed yet, will try archiving it next time 2023-10-02T12:25:19,986 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta:.meta(num 1696249503264) roll requested 2023-10-02T12:25:19,988 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.1696249503214 not finished, retry = 0 2023-10-02T12:25:19,998 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta.1696249519986.meta, exclude list is [], retry=0 2023-10-02T12:25:20,002 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38869,DS-d3b004e5-e24b-4408-a9bf-421ca1308ded,DISK] 2023-10-02T12:25:20,004 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta.1696249503264.meta with entries=20, filesize=4.92 KB; new WAL /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta.1696249519986.meta 2023-10-02T12:25:20,004 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38869,DS-d3b004e5-e24b-4408-a9bf-421ca1308ded,DISK]] 2023-10-02T12:25:20,004 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta.1696249503264.meta is not closed yet, will try archiving it next time 2023-10-02T12:25:20,008 DEBUG [rs(jenkins-hbase3.apache.org,35033,1696249482294)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-10-02T12:25:20,009 INFO [rs(jenkins-hbase3.apache.org,35033,1696249482294)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696249519968 on jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:20,020 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:25:20,020 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,35033,1696249482294' in zk 2023-10-02T12:25:20,024 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:25:20,024 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:25:20,024 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:25:20,024 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:20,025 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:25:20,025 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:25:20,025 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:25:20,025 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:25:20,026 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:25:20,026 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:20,026 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:25:20,027 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:25:20,027 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:20,028 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,35033,1696249482294': 2023-10-02T12:25:20,028 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,35033,1696249482294' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:25:20,028 INFO [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:25:20,028 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:25:20,028 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:25:20,028 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:25:20,028 INFO [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:25:20,030 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:25:20,030 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:25:20,030 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:25:20,030 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:25:20,030 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:25:20,031 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:25:20,031 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:25:20,031 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:25:20,031 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:25:20,031 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:25:20,032 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:25:20,032 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:25:20,032 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:20,033 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:25:20,033 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:25:20,034 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:20,039 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:25:20,039 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:25:20,039 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:25:20,040 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:25:20,040 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:25:20,040 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:25:20,040 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:25:20,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:25:20,145 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249520144_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } 2023-10-02T12:25:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249520145 (current time:1696249520145). 2023-10-02T12:25:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:25:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249520144_default_test-1696249492588 VERSION not specified, setting to 2 2023-10-02T12:25:20,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5be17442, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:25:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:25:20,157 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50150, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:25:20,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:25:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=56, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1696249520144_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } 2023-10-02T12:25:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249520144_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 }, snapshot procedure id = 56 2023-10-02T12:25:20,162 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1696249520144_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:25:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-10-02T12:25:20,163 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1696249520144_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:25:20,165 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot, current working dir path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_1696249520144_default_test-1696249492588 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:25:20,167 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1696249520144_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:25:20,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-10-02T12:25:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-10-02T12:25:20,574 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1696249520144_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:25:20,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE; SnapshotRegionProcedure ff520da84fd90f4138d4152604b554a1}] 2023-10-02T12:25:20,577 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=57, ppid=56, state=RUNNABLE; SnapshotRegionProcedure ff520da84fd90f4138d4152604b554a1 2023-10-02T12:25:20,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35033 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=57 2023-10-02T12:25:20,729 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. 2023-10-02T12:25:20,729 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.HRegion(2520): Flush status journal for ff520da84fd90f4138d4152604b554a1: 2023-10-02T12:25:20,729 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. for snapshot_1696249520144_default_test-1696249492588 completed. 2023-10-02T12:25:20,729 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(240): Storing 'test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1.' region-info for snapshot=snapshot_1696249520144_default_test-1696249492588 2023-10-02T12:25:20,729 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:25:20,730 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1/f/fdcfc2b4f61244028e163a86e4ae5481] hfiles 2023-10-02T12:25:20,730 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1/f/fdcfc2b4f61244028e163a86e4ae5481 for snapshot=snapshot_1696249520144_default_test-1696249492588 2023-10-02T12:25:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-10-02T12:25:21,138 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. 2023-10-02T12:25:21,138 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=57 2023-10-02T12:25:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster(3995): Remote procedure done, pid=57 2023-10-02T12:25:21,138 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249520144_default_test-1696249492588 on region ff520da84fd90f4138d4152604b554a1 2023-10-02T12:25:21,140 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=57, ppid=56, state=RUNNABLE; SnapshotRegionProcedure ff520da84fd90f4138d4152604b554a1 2023-10-02T12:25:21,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=57, resume processing ppid=56 2023-10-02T12:25:21,142 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=57, ppid=56, state=SUCCESS; SnapshotRegionProcedure ff520da84fd90f4138d4152604b554a1 in 567 msec 2023-10-02T12:25:21,142 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1696249520144_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:25:21,143 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1696249520144_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:25:21,144 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1696249520144_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:25:21,144 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249520144_default_test-1696249492588 2023-10-02T12:25:21,145 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_1696249520144_default_test-1696249492588 2023-10-02T12:25:21,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-10-02T12:25:21,556 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1696249520144_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:25:21,568 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1696249520144_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:25:21,569 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_1696249520144_default_test-1696249492588 to hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/snapshot_1696249520144_default_test-1696249492588 2023-10-02T12:25:21,570 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=56, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1696249520144_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:25:21,571 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249520144_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 }, snapshot procedure id = 56 2023-10-02T12:25:21,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=56, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=56, snapshot={ ss=snapshot_1696249520144_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } in 1.4120 sec 2023-10-02T12:25:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=56 2023-10-02T12:25:22,271 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696249517787 2023-10-02T12:25:22,271 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:25:22,271 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-10-02T12:25:22,271 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249517787_test-1696249492588 2023-10-02T12:25:22,271 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249520144_default_test-1696249492588 to hdfs://localhost:36497/backupUT/backup_1696249517787/default/test-1696249492588/ 2023-10-02T12:25:22,271 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:25:22,307 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:36497, inputRoot=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a 2023-10-02T12:25:22,307 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43611,1696249478718_-854644318_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36497/backupUT/backup_1696249517787/default/test-1696249492588, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36497/backupUT/backup_1696249517787/default/test-1696249492588/.hbase-snapshot/.tmp/snapshot_1696249520144_default_test-1696249492588 2023-10-02T12:25:22,307 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:36497, inputRoot=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a, snapshotDir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/snapshot_1696249520144_default_test-1696249492588. 2023-10-02T12:25:22,326 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/snapshot_1696249520144_default_test-1696249492588 to hdfs://localhost:36497/backupUT/backup_1696249517787/default/test-1696249492588/.hbase-snapshot/.tmp/snapshot_1696249520144_default_test-1696249492588 2023-10-02T12:25:22,767 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:22,767 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:22,768 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:22,768 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:23,875 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/hadoop-689510728329091161.jar 2023-10-02T12:25:23,875 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:23,876 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:23,876 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:23,877 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:23,877 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:23,878 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:23,878 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:23,878 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:23,879 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:23,880 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:25:23,880 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:25:23,881 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:25:23,881 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:25:23,881 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:25:23,882 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:25:23,882 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:25:23,883 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:25:23,883 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:25:23,884 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:25:23,884 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:25:23,884 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:25:23,885 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:25:23,886 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:25:23,886 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:25:23,887 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:25:23,887 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:25:23,888 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:25:23,888 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:25:23,938 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249520144_default_test-1696249492588' hfile list 2023-10-02T12:25:23,941 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-10-02T12:25:24,067 WARN [Listener at jenkins-hbase3.apache.org/37677 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/mapreduce.cluster.local.dir/1696249524009/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:25:24,067 WARN [Listener at jenkins-hbase3.apache.org/37677 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/mapreduce.cluster.local.dir/1696249524009/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:25:24,155 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:25:24,183 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1/f/fdcfc2b4f61244028e163a86e4ae5481 output=hdfs://localhost:36497/backupUT/backup_1696249517787/default/test-1696249492588/archive/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1/f/fdcfc2b4f61244028e163a86e4ae5481 2023-10-02T12:25:24,184 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-10-02T12:25:25,121 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:25:25,122 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:25:25,131 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249520144_default_test-1696249492588 2023-10-02T12:25:25,131 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249520144_default_test-1696249492588 finished. 2023-10-02T12:25:25,131 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1034): test-1696249492588 2023-10-02T12:25:25,140 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249517787 2023-10-02T12:25:25,140 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:25:25,549 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:36497/backupUT/backup_1696249517787/.backup.manifest 2023-10-02T12:25:25,549 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(358): Backup backup_1696249517787 finished: type=FULL,tablelist=test-1696249492588,targetRootDir=hdfs://localhost:36497/backupUT,startts=1696249519945,completets=1696249525140,bytescopied=0 2023-10-02T12:25:25,549 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:25:25,549 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249520144_default_test-1696249492588 2023-10-02T12:25:25,550 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249520144_default_test-1696249492588" 2023-10-02T12:25:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249520144_default_test-1696249492588 2023-10-02T12:25:25,554 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249520144_default_test-1696249492588 for backup backup_1696249517787 succeeded. 2023-10-02T12:25:25,555 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:25:25,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:25:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:25:25,566 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:25:25,567 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:25:25,573 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(376): Backup backup_1696249517787 completed. 2023-10-02T12:25:25,574 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:25,588 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] backup.TestBackupDeleteWithFailures(147): backup complete 2023-10-02T12:25:25,612 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:25:25,623 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:25:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249525623 (current time:1696249525623). 2023-10-02T12:25:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:25:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:25:25,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c0a74e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:25:25,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:25:25,635 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55926, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:25:25,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:25,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:25:25,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=58, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:25:25,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 58 2023-10-02T12:25:25,638 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:25:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-10-02T12:25:25,639 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:25:25,642 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot, current working dir path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:25:25,644 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:25:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-10-02T12:25:25,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-10-02T12:25:26,054 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:25:26,055 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30}] 2023-10-02T12:25:26,057 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=59, ppid=58, state=RUNNABLE; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:26,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35033 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=59 2023-10-02T12:25:26,209 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:26,209 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2819): Flushing 3dfc58254ae9ba5ba94c6dcd551aba30 2/2 column families, dataSize=1.06 KB heapSize=2.22 KB 2023-10-02T12:25:26,213 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/6b509d0f4f4f4290bdfe10ec96f6e0f3 is 169, key is trslm:hdfs://localhost:36497/backupUT\x00test-1696249492588/meta:log-roll-map/1696249525135/Put/seqid=0 2023-10-02T12:25:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-10-02T12:25:26,618 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=511 B at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/6b509d0f4f4f4290bdfe10ec96f6e0f3 2023-10-02T12:25:26,627 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/b7119e7f561746f787395f659b602c5b is 310, key is session:backup_1696249517787/session:context/1696249525566/Put/seqid=0 2023-10-02T12:25:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-10-02T12:25:26,803 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-10-02T12:25:26,803 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-10-02T12:25:27,032 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=579 B at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/b7119e7f561746f787395f659b602c5b 2023-10-02T12:25:27,039 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/6b509d0f4f4f4290bdfe10ec96f6e0f3 as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/6b509d0f4f4f4290bdfe10ec96f6e0f3 2023-10-02T12:25:27,045 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/6b509d0f4f4f4290bdfe10ec96f6e0f3, entries=5, sequenceid=57, filesize=5.5 K 2023-10-02T12:25:27,046 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/b7119e7f561746f787395f659b602c5b as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/b7119e7f561746f787395f659b602c5b 2023-10-02T12:25:27,052 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/b7119e7f561746f787395f659b602c5b, entries=2, sequenceid=57, filesize=5.3 K 2023-10-02T12:25:27,053 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(3022): Finished flush of dataSize ~1.06 KB/1090, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 3dfc58254ae9ba5ba94c6dcd551aba30 in 844ms, sequenceid=57, compaction requested=true 2023-10-02T12:25:27,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.HRegion(2520): Flush status journal for 3dfc58254ae9ba5ba94c6dcd551aba30: 2023-10-02T12:25:27,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. for snapshot_backup_system completed. 2023-10-02T12:25:27,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:25:27,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:25:27,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/44756fad2d4c4c749f6666e97cd9aa7a, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/ca7ddbd11d4149f288b81612e951712f, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/6b509d0f4f4f4290bdfe10ec96f6e0f3] hfiles 2023-10-02T12:25:27,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/44756fad2d4c4c749f6666e97cd9aa7a for snapshot=snapshot_backup_system 2023-10-02T12:25:27,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/ca7ddbd11d4149f288b81612e951712f for snapshot=snapshot_backup_system 2023-10-02T12:25:27,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/6b509d0f4f4f4290bdfe10ec96f6e0f3 for snapshot=snapshot_backup_system 2023-10-02T12:25:27,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/9f3d5d24e61b47dca538b66fb59a927a, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/3ba88e6f8052440dbf83bfb0e7a50568, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/06206d275c734da89d8139ad46731eb7, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/b7119e7f561746f787395f659b602c5b] hfiles 2023-10-02T12:25:27,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/9f3d5d24e61b47dca538b66fb59a927a for snapshot=snapshot_backup_system 2023-10-02T12:25:27,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/3ba88e6f8052440dbf83bfb0e7a50568 for snapshot=snapshot_backup_system 2023-10-02T12:25:27,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/06206d275c734da89d8139ad46731eb7 for snapshot=snapshot_backup_system 2023-10-02T12:25:27,054 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/b7119e7f561746f787395f659b602c5b for snapshot=snapshot_backup_system 2023-10-02T12:25:27,461 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:27,461 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=59 2023-10-02T12:25:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster(3995): Remote procedure done, pid=59 2023-10-02T12:25:27,462 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:27,463 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=59, ppid=58, state=RUNNABLE; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:27,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=59, resume processing ppid=58 2023-10-02T12:25:27,466 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:25:27,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=59, ppid=58, state=SUCCESS; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30 in 1.4090 sec 2023-10-02T12:25:27,466 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:25:27,467 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:25:27,467 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:25:27,468 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:25:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-10-02T12:25:27,878 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:25:27,887 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:25:27,887 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:25:27,889 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=58, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:25:27,889 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 58 2023-10-02T12:25:27,890 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=58, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=58, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2530 sec 2023-10-02T12:25:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=58 2023-10-02T12:25:29,753 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupAdminImpl(227): Deleting backup backup_1696249517787 ... 2023-10-02T12:25:29,754 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-10-02T12:25:29,755 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] util.BackupUtils(435): Delete log files: default 2023-10-02T12:25:29,756 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1696249517787 2023-10-02T12:25:29,757 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:36497/backupUT/backup_1696249517787/default/test-1696249492588. 2023-10-02T12:25:29,757 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] util.BackupUtils(465): hdfs://localhost:36497/backupUT/backup_1696249517787/default is empty, remove it. 2023-10-02T12:25:29,761 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-10-02T12:25:29,761 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-10-02T12:25:29,763 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupAdminImpl(276): Delete backup backup_1696249517787 completed. 2023-10-02T12:25:29,771 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1034): test-1696249492588 2023-10-02T12:25:29,773 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:25:29,780 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:25:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:25:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: postDeleteSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.postDeleteSnapshot(TestBackupDeleteWithFailures.java:111) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$89.call(MasterCoprocessorHost.java:1164) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$89.call(MasterCoprocessorHost.java:1161) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.postDeleteSnapshot(MasterCoprocessorHost.java:1161) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:388) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:762) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-10-02T12:25:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.CallRunner(138): callId: 274 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:35550 deadline: 1696249589780, exception=java.io.IOException: postDeleteSnapshot 2023-10-02T12:25:29,885 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:25:29,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException: Snapshot 'snapshot_backup_system' doesn't exist on the filesystem at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:361) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:762) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-10-02T12:25:29,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.CallRunner(138): callId: 276 service: MasterService methodName: DeleteSnapshot size: 57 connection: 172.31.12.81:35550 deadline: 1696249589885, exception=org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException: Snapshot 'snapshot_backup_system' doesn't exist on the filesystem 2023-10-02T12:25:29,892 WARN [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupAdminImpl(163): Delete operation succeeded, there were some errors: org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException: org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException: Snapshot 'snapshot_backup_system' doesn't exist on the filesystem at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:361) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:762) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:377) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:365) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.MasterCallable.call(MasterCallable.java:101) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3105) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.executeCallable(HBaseAdmin.java:3097) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.client.HBaseAdmin.deleteSnapshot(HBaseAdmin.java:3005) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.backup.impl.BackupSystemTable.deleteSnapshot(BackupSystemTable.java:1576) ~[classes/:?] at org.apache.hadoop.hbase.backup.impl.BackupAdminImpl.deleteBackups(BackupAdminImpl.java:149) ~[classes/:?] at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures.testBackupDeleteWithFailuresAfter(TestBackupDeleteWithFailures.java:161) ~[test-classes/:?] at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures.testBackupDeleteWithFailures(TestBackupDeleteWithFailures.java:137) ~[test-classes/:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException: Snapshot 'snapshot_backup_system' doesn't exist on the filesystem at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.deleteSnapshot(SnapshotManager.java:361) at org.apache.hadoop.hbase.master.MasterRpcServices.deleteSnapshot(MasterRpcServices.java:762) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:391) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:93) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:429) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:424) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:197) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:218) ~[hbase-client-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] ... 1 more 2023-10-02T12:25:29,894 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:25:29,907 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e74f8cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:25:29,916 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67a88934, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:25:29,917 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:25:29,919 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60816, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:25:29,923 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:25:29,924 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55942, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService REPAIR status: no failed sessions found. Checking failed delete backup operation ... No failed backup DELETE operation found 2023-10-02T12:25:29,945 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:25:29,950 ERROR [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1579): Snapshot snapshot_backup_system does not exists No failed backup MERGE operation found 2023-10-02T12:25:29,951 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:25:29,955 ERROR [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1579): Snapshot snapshot_backup_system does not exists 2023-10-02T12:25:29,956 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:29,956 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:29,960 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] backup.TestBackupDeleteWithFailures(143): test repair backup delete on a single table with data and failures PRE_SNAPSHOT_FAILURE 2023-10-02T12:25:29,969 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ce96ae4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:25:29,973 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:25:29,975 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55946, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:25:29,977 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:25:29,978 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60832, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:25:29,994 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:25:30,004 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:25:30,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249530004 (current time:1696249530004). 2023-10-02T12:25:30,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:25:30,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:25:30,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31741a60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:25:30,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:25:30,017 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55948, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:25:30,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:30,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:25:30,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:25:30,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2023-10-02T12:25:30,022 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:25:30,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-10-02T12:25:30,023 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:25:30,026 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot, current working dir path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:25:30,029 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:25:30,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-10-02T12:25:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-10-02T12:25:30,437 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:25:30,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30}] 2023-10-02T12:25:30,439 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:30,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35033 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2023-10-02T12:25:30,591 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:30,591 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2819): Flushing 3dfc58254ae9ba5ba94c6dcd551aba30 2/2 column families, dataSize=268 B heapSize=1.13 KB 2023-10-02T12:25:30,596 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/3da00ee715b64e8aa061315d019c0b09 is 91, key is incrbackupset:hdfs://localhost:36497/backupUT/meta:test-1696249492588/1696249529771/Put/seqid=0 2023-10-02T12:25:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-10-02T12:25:31,001 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=124 B at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/3da00ee715b64e8aa061315d019c0b09 2023-10-02T12:25:31,007 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3da00ee715b64e8aa061315d019c0b09 2023-10-02T12:25:31,010 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/2efcd96b58334c92986802f8ecd44bea is 59, key is session:backup_1696249517787/session:/1696249529762/DeleteFamily/seqid=0 2023-10-02T12:25:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-10-02T12:25:31,416 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=144 B at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/2efcd96b58334c92986802f8ecd44bea 2023-10-02T12:25:31,422 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2efcd96b58334c92986802f8ecd44bea 2023-10-02T12:25:31,423 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/3da00ee715b64e8aa061315d019c0b09 as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/3da00ee715b64e8aa061315d019c0b09 2023-10-02T12:25:31,429 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3da00ee715b64e8aa061315d019c0b09 2023-10-02T12:25:31,429 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/3da00ee715b64e8aa061315d019c0b09, entries=2, sequenceid=65, filesize=5.2 K 2023-10-02T12:25:31,431 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/2efcd96b58334c92986802f8ecd44bea as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/2efcd96b58334c92986802f8ecd44bea 2023-10-02T12:25:31,436 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2efcd96b58334c92986802f8ecd44bea 2023-10-02T12:25:31,436 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/2efcd96b58334c92986802f8ecd44bea, entries=2, sequenceid=65, filesize=5.2 K 2023-10-02T12:25:31,437 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3022): Finished flush of dataSize ~268 B/268, heapSize ~1.09 KB/1120, currentSize=0 B/0 for 3dfc58254ae9ba5ba94c6dcd551aba30 in 846ms, sequenceid=65, compaction requested=true 2023-10-02T12:25:31,437 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2520): Flush status journal for 3dfc58254ae9ba5ba94c6dcd551aba30: 2023-10-02T12:25:31,437 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. for snapshot_backup_system completed. 2023-10-02T12:25:31,438 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:25:31,438 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:25:31,438 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/44756fad2d4c4c749f6666e97cd9aa7a, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/ca7ddbd11d4149f288b81612e951712f, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/6b509d0f4f4f4290bdfe10ec96f6e0f3, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/3da00ee715b64e8aa061315d019c0b09] hfiles 2023-10-02T12:25:31,438 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/44756fad2d4c4c749f6666e97cd9aa7a for snapshot=snapshot_backup_system 2023-10-02T12:25:31,438 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/ca7ddbd11d4149f288b81612e951712f for snapshot=snapshot_backup_system 2023-10-02T12:25:31,438 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/6b509d0f4f4f4290bdfe10ec96f6e0f3 for snapshot=snapshot_backup_system 2023-10-02T12:25:31,438 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/3da00ee715b64e8aa061315d019c0b09 for snapshot=snapshot_backup_system 2023-10-02T12:25:31,438 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/9f3d5d24e61b47dca538b66fb59a927a, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/3ba88e6f8052440dbf83bfb0e7a50568, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/06206d275c734da89d8139ad46731eb7, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/b7119e7f561746f787395f659b602c5b, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/2efcd96b58334c92986802f8ecd44bea] hfiles 2023-10-02T12:25:31,438 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/9f3d5d24e61b47dca538b66fb59a927a for snapshot=snapshot_backup_system 2023-10-02T12:25:31,438 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/3ba88e6f8052440dbf83bfb0e7a50568 for snapshot=snapshot_backup_system 2023-10-02T12:25:31,438 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/06206d275c734da89d8139ad46731eb7 for snapshot=snapshot_backup_system 2023-10-02T12:25:31,438 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/b7119e7f561746f787395f659b602c5b for snapshot=snapshot_backup_system 2023-10-02T12:25:31,438 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/2efcd96b58334c92986802f8ecd44bea for snapshot=snapshot_backup_system 2023-10-02T12:25:31,846 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:31,846 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=61 2023-10-02T12:25:31,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster(3995): Remote procedure done, pid=61 2023-10-02T12:25:31,847 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:31,848 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:31,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=61, resume processing ppid=60 2023-10-02T12:25:31,850 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:25:31,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=61, ppid=60, state=SUCCESS; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30 in 1.4120 sec 2023-10-02T12:25:31,851 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:25:31,852 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:25:31,852 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:25:31,852 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:25:32,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-10-02T12:25:32,266 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:25:32,277 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:25:32,278 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:25:32,279 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:25:32,279 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2023-10-02T12:25:32,280 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=60, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2590 sec 2023-10-02T12:25:34,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-10-02T12:25:34,130 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(115): Backup backup_1696249529969 started at 1696249534130. 2023-10-02T12:25:34,132 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(119): Backup session backup_1696249529969 has been started. 2023-10-02T12:25:34,134 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:25:34,134 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:25:34,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:25:34,135 INFO [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:25:34,135 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:25:34,136 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:25:34,136 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:25:34,137 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:25:34,138 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:34,138 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:25:34,138 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:25:34,138 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:25:34,138 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:25:34,139 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:25:34,139 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:25:34,139 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:25:34,139 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:25:34,139 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:25:34,140 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:25:34,140 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:25:34,141 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:25:34,141 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:25:34,141 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,35033,1696249482294' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:25:34,143 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:25:34,143 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:34,143 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:25:34,143 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:25:34,143 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:25:34,143 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:25:34,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:25:34,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:25:34,144 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:34,145 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:25:34,145 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,35033,1696249482294' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:25:34,145 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@7f1aa62b[Count = 0] remaining members to acquire global barrier 2023-10-02T12:25:34,145 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:25:34,145 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:25:34,146 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:25:34,146 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:25:34,147 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:25:34,147 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:25:34,147 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:25:34,147 DEBUG [rs(jenkins-hbase3.apache.org,35033,1696249482294)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:34,147 INFO [rs(jenkins-hbase3.apache.org,35033,1696249482294)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696249519968 highest: 1696249519968 on jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:34,147 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C35033%2C1696249482294:(num 1696249519968) roll requested 2023-10-02T12:25:34,164 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.1696249534147, exclude list is [], retry=0 2023-10-02T12:25:34,169 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38869,DS-d3b004e5-e24b-4408-a9bf-421ca1308ded,DISK] 2023-10-02T12:25:34,172 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.1696249519968 with entries=18, filesize=3.44 KB; new WAL /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.1696249534147 2023-10-02T12:25:34,172 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38869,DS-d3b004e5-e24b-4408-a9bf-421ca1308ded,DISK]] 2023-10-02T12:25:34,172 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.1696249519968 is not closed yet, will try archiving it next time 2023-10-02T12:25:34,173 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta:.meta(num 1696249519986) roll requested 2023-10-02T12:25:34,173 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.1696249503214 to hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/oldWALs/jenkins-hbase3.apache.org%2C35033%2C1696249482294.1696249503214 2023-10-02T12:25:34,190 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta.1696249534173.meta, exclude list is [], retry=0 2023-10-02T12:25:34,194 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38869,DS-d3b004e5-e24b-4408-a9bf-421ca1308ded,DISK] 2023-10-02T12:25:34,195 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta.1696249519986.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta.1696249534173.meta 2023-10-02T12:25:34,195 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:38869,DS-d3b004e5-e24b-4408-a9bf-421ca1308ded,DISK]] 2023-10-02T12:25:34,196 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta.1696249519986.meta is not closed yet, will try archiving it next time 2023-10-02T12:25:34,200 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/WALs/jenkins-hbase3.apache.org,35033,1696249482294/jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta.1696249519986.meta to hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/oldWALs/jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta.1696249519986.meta 2023-10-02T12:25:34,208 DEBUG [rs(jenkins-hbase3.apache.org,35033,1696249482294)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-10-02T12:25:34,208 INFO [rs(jenkins-hbase3.apache.org,35033,1696249482294)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696249534147 on jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:34,219 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:25:34,219 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,35033,1696249482294' in zk 2023-10-02T12:25:34,222 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:25:34,222 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:34,222 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:25:34,222 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:25:34,222 DEBUG [member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:25:34,223 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:25:34,224 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:25:34,224 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:25:34,224 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:25:34,224 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:34,225 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:25:34,225 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:25:34,225 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:34,226 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,35033,1696249482294': 2023-10-02T12:25:34,226 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,35033,1696249482294' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:25:34,226 INFO [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:25:34,226 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:25:34,226 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:25:34,226 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:25:34,226 INFO [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:25:34,228 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:25:34,228 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:25:34,228 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:25:34,228 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:25:34,228 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:25:34,229 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:25:34,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:25:34,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:25:34,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:25:34,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:25:34,230 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:25:34,230 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:25:34,231 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:34,231 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:25:34,232 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:25:34,232 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:34,236 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:25:34,236 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:25:34,236 DEBUG [(jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:25:34,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:25:34,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:25:34,237 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:25:34,237 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:25:34,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:25:34,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249534340_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } 2023-10-02T12:25:34,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249534341 (current time:1696249534341). 2023-10-02T12:25:34,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:25:34,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249534340_default_test-1696249492588 VERSION not specified, setting to 2 2023-10-02T12:25:34,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad23647, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:25:34,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:25:34,353 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54714, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:25:34,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:34,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:25:34,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=62, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249534340_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } 2023-10-02T12:25:34,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249534340_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-10-02T12:25:34,357 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249534340_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:25:34,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-10-02T12:25:34,358 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249534340_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:25:34,361 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot, current working dir path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_1696249534340_default_test-1696249492588 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:25:34,364 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249534340_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:25:34,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-10-02T12:25:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-10-02T12:25:34,770 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249534340_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:25:34,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure ff520da84fd90f4138d4152604b554a1}] 2023-10-02T12:25:34,774 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure ff520da84fd90f4138d4152604b554a1 2023-10-02T12:25:34,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35033 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=63 2023-10-02T12:25:34,926 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. 2023-10-02T12:25:34,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2520): Flush status journal for ff520da84fd90f4138d4152604b554a1: 2023-10-02T12:25:34,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. for snapshot_1696249534340_default_test-1696249492588 completed. 2023-10-02T12:25:34,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(240): Storing 'test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1.' region-info for snapshot=snapshot_1696249534340_default_test-1696249492588 2023-10-02T12:25:34,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:25:34,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1/f/fdcfc2b4f61244028e163a86e4ae5481] hfiles 2023-10-02T12:25:34,927 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1/f/fdcfc2b4f61244028e163a86e4ae5481 for snapshot=snapshot_1696249534340_default_test-1696249492588 2023-10-02T12:25:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-10-02T12:25:35,337 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. 2023-10-02T12:25:35,337 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=63 2023-10-02T12:25:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster(3995): Remote procedure done, pid=63 2023-10-02T12:25:35,338 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249534340_default_test-1696249492588 on region ff520da84fd90f4138d4152604b554a1 2023-10-02T12:25:35,340 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure ff520da84fd90f4138d4152604b554a1 2023-10-02T12:25:35,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=63, resume processing ppid=62 2023-10-02T12:25:35,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=63, ppid=62, state=SUCCESS; SnapshotRegionProcedure ff520da84fd90f4138d4152604b554a1 in 571 msec 2023-10-02T12:25:35,343 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249534340_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:25:35,344 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249534340_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:25:35,344 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249534340_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:25:35,345 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249534340_default_test-1696249492588 2023-10-02T12:25:35,346 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_1696249534340_default_test-1696249492588 2023-10-02T12:25:35,356 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249534340_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:25:35,362 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249534340_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:25:35,363 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_1696249534340_default_test-1696249492588 to hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/snapshot_1696249534340_default_test-1696249492588 2023-10-02T12:25:35,364 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249534340_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:25:35,364 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249534340_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-10-02T12:25:35,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=62, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249534340_default_test-1696249492588 table=test-1696249492588 type=FLUSH ttl=0 } in 1.0100 sec 2023-10-02T12:25:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-10-02T12:25:35,463 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696249529969 2023-10-02T12:25:35,463 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:25:35,463 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-10-02T12:25:35,463 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249529969_test-1696249492588 2023-10-02T12:25:35,463 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249534340_default_test-1696249492588 to hdfs://localhost:36497/backupUT/backup_1696249529969/default/test-1696249492588/ 2023-10-02T12:25:35,464 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:25:35,491 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:36497, inputRoot=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a 2023-10-02T12:25:35,491 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,43611,1696249478718_-295903056_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:36497/backupUT/backup_1696249529969/default/test-1696249492588, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:36497/backupUT/backup_1696249529969/default/test-1696249492588/.hbase-snapshot/.tmp/snapshot_1696249534340_default_test-1696249492588 2023-10-02T12:25:35,492 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:36497, inputRoot=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a, snapshotDir=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/snapshot_1696249534340_default_test-1696249492588. 2023-10-02T12:25:35,514 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/snapshot_1696249534340_default_test-1696249492588 to hdfs://localhost:36497/backupUT/backup_1696249529969/default/test-1696249492588/.hbase-snapshot/.tmp/snapshot_1696249534340_default_test-1696249492588 2023-10-02T12:25:35,957 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:35,958 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:35,959 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:35,959 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:37,080 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/hadoop-7535637731327591739.jar 2023-10-02T12:25:37,081 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:37,081 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:37,082 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:37,083 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:37,083 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:37,083 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:37,084 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:37,084 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:37,085 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:25:37,086 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:25:37,087 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:25:37,087 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:25:37,088 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:25:37,088 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:25:37,089 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:25:37,089 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:25:37,090 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:25:37,091 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:25:37,091 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:25:37,092 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:25:37,092 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:25:37,093 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:25:37,093 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:25:37,094 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:25:37,095 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:25:37,095 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:25:37,096 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:25:37,097 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:25:37,149 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249534340_default_test-1696249492588' hfile list 2023-10-02T12:25:37,152 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-10-02T12:25:37,292 WARN [Listener at jenkins-hbase3.apache.org/37677 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/mapreduce.cluster.local.dir/1696249537223/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:25:37,292 WARN [Listener at jenkins-hbase3.apache.org/37677 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/mapreduce.cluster.local.dir/1696249537223/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:25:37,358 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:25:37,386 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1/f/fdcfc2b4f61244028e163a86e4ae5481 output=hdfs://localhost:36497/backupUT/backup_1696249529969/default/test-1696249492588/archive/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1/f/fdcfc2b4f61244028e163a86e4ae5481 2023-10-02T12:25:37,386 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-10-02T12:25:38,335 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:25:38,337 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:25:38,346 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249534340_default_test-1696249492588 2023-10-02T12:25:38,346 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249534340_default_test-1696249492588 finished. 2023-10-02T12:25:38,346 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1034): test-1696249492588 2023-10-02T12:25:38,355 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249529969 2023-10-02T12:25:38,355 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:25:38,770 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:36497/backupUT/backup_1696249529969/.backup.manifest 2023-10-02T12:25:38,770 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(358): Backup backup_1696249529969 finished: type=FULL,tablelist=test-1696249492588,targetRootDir=hdfs://localhost:36497/backupUT,startts=1696249534130,completets=1696249538354,bytescopied=0 2023-10-02T12:25:38,770 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:25:38,770 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249534340_default_test-1696249492588 2023-10-02T12:25:38,772 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249534340_default_test-1696249492588" 2023-10-02T12:25:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249534340_default_test-1696249492588 2023-10-02T12:25:38,776 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249534340_default_test-1696249492588 for backup backup_1696249529969 succeeded. 2023-10-02T12:25:38,777 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:25:38,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:25:38,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:25:38,789 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:25:38,791 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:25:38,798 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.TableBackupClient(376): Backup backup_1696249529969 completed. 2023-10-02T12:25:38,799 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:38,820 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] backup.TestBackupDeleteWithFailures(147): backup complete 2023-10-02T12:25:38,864 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:25:38,881 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:25:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249538881 (current time:1696249538881). 2023-10-02T12:25:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:25:38,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:25:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24c3d882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:25:38,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:25:38,906 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54726, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:25:38,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:25:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:25:38,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-10-02T12:25:38,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-10-02T12:25:38,911 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:25:38,912 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:25:38,916 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot, current working dir path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:25:38,919 WARN [PEWorker-5 {}] procedure.SnapshotProcedure(200): unexpected exception while execute pid=64, state=FAILED:SNAPSHOT_PRE_OPERATION, locked=true, exception=java.io.IOException via master-snapshot:java.io.IOException: preSnapshot; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }. Mark procedure Failed. java.io.IOException: preSnapshot at org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver.preSnapshot(TestBackupDeleteWithFailures.java:95) ~[test-classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$79.call(MasterCoprocessorHost.java:1068) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost$79.call(MasterCoprocessorHost.java:1065) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost$ObserverOperationWithoutResult.callObserver(CoprocessorHost.java:558) ~[classes/:?] at org.apache.hadoop.hbase.coprocessor.CoprocessorHost.execOperation(CoprocessorHost.java:631) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterCoprocessorHost.preSnapshot(MasterCoprocessorHost.java:1065) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:306) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] 2023-10-02T12:25:38,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1531): Rolled back pid=64, state=ROLLEDBACK, exception=java.io.IOException via master-snapshot:java.io.IOException: preSnapshot; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } exec-time=33 msec 2023-10-02T12:25:39,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-10-02T12:25:39,052 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@281d0419, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:25:39,070 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c09f3e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:25:39,073 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:25:39,074 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33116, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:25:39,083 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:25:39,085 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54742, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService REPAIR status: no failed sessions found. Checking failed delete backup operation ... Found failed DELETE operation for: backup_1696249529969 Running DELETE again ... 2023-10-02T12:25:39,101 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1538): Restoring backup:system from snapshot 2023-10-02T12:25:39,107 WARN [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1550): Could not restore backup system table. Snapshot snapshot_backup_system does not exists. 2023-10-02T12:25:39,107 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:25:39,123 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:25:39,139 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:25:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249539139 (current time:1696249539139). 2023-10-02T12:25:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:25:39,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:25:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fb0d95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:25:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:25:39,161 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54754, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:25:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:25:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure2.ProcedureExecutor(1032): Stored pid=65, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:25:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 65 2023-10-02T12:25:39,168 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:25:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=65 2023-10-02T12:25:39,170 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:25:39,174 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot, current working dir path: hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor53.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:25:39,176 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:25:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=65 2023-10-02T12:25:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=65 2023-10-02T12:25:39,587 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:25:39,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30}] 2023-10-02T12:25:39,589 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=66, ppid=65, state=RUNNABLE; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:39,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35033 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=66 2023-10-02T12:25:39,741 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:39,741 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HRegion(2819): Flushing 3dfc58254ae9ba5ba94c6dcd551aba30 2/2 column families, dataSize=1.21 KB heapSize=2.59 KB 2023-10-02T12:25:39,746 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/05e4f6500d514dac8f9b7c3886b9cfae is 169, key is trslm:hdfs://localhost:36497/backupUT\x00test-1696249492588/meta:log-roll-map/1696249538349/Put/seqid=0 2023-10-02T12:25:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=65 2023-10-02T12:25:40,152 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=571 B at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/05e4f6500d514dac8f9b7c3886b9cfae 2023-10-02T12:25:40,162 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/a6a3e92419324310b9bf4e893b2219dc is 310, key is session:backup_1696249529969/session:context/1696249538790/Put/seqid=0 2023-10-02T12:25:40,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=65 2023-10-02T12:25:40,567 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=668 B at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/a6a3e92419324310b9bf4e893b2219dc 2023-10-02T12:25:40,576 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/05e4f6500d514dac8f9b7c3886b9cfae as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/05e4f6500d514dac8f9b7c3886b9cfae 2023-10-02T12:25:40,583 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/05e4f6500d514dac8f9b7c3886b9cfae, entries=5, sequenceid=80, filesize=5.5 K 2023-10-02T12:25:40,585 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/a6a3e92419324310b9bf4e893b2219dc as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/a6a3e92419324310b9bf4e893b2219dc 2023-10-02T12:25:40,593 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/a6a3e92419324310b9bf4e893b2219dc, entries=2, sequenceid=80, filesize=5.3 K 2023-10-02T12:25:40,594 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HRegion(3022): Finished flush of dataSize ~1.21 KB/1239, heapSize ~2.55 KB/2616, currentSize=0 B/0 for 3dfc58254ae9ba5ba94c6dcd551aba30 in 853ms, sequenceid=80, compaction requested=true 2023-10-02T12:25:40,594 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.HRegion(2520): Flush status journal for 3dfc58254ae9ba5ba94c6dcd551aba30: 2023-10-02T12:25:40,594 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. for snapshot_backup_system completed. 2023-10-02T12:25:40,594 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:25:40,595 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:25:40,595 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/44756fad2d4c4c749f6666e97cd9aa7a, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/ca7ddbd11d4149f288b81612e951712f, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/6b509d0f4f4f4290bdfe10ec96f6e0f3, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/3da00ee715b64e8aa061315d019c0b09, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/05e4f6500d514dac8f9b7c3886b9cfae] hfiles 2023-10-02T12:25:40,595 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/44756fad2d4c4c749f6666e97cd9aa7a for snapshot=snapshot_backup_system 2023-10-02T12:25:40,595 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/ca7ddbd11d4149f288b81612e951712f for snapshot=snapshot_backup_system 2023-10-02T12:25:40,595 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/6b509d0f4f4f4290bdfe10ec96f6e0f3 for snapshot=snapshot_backup_system 2023-10-02T12:25:40,595 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/3da00ee715b64e8aa061315d019c0b09 for snapshot=snapshot_backup_system 2023-10-02T12:25:40,595 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/05e4f6500d514dac8f9b7c3886b9cfae for snapshot=snapshot_backup_system 2023-10-02T12:25:40,595 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/9f3d5d24e61b47dca538b66fb59a927a, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/3ba88e6f8052440dbf83bfb0e7a50568, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/06206d275c734da89d8139ad46731eb7, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/b7119e7f561746f787395f659b602c5b, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/2efcd96b58334c92986802f8ecd44bea, hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/a6a3e92419324310b9bf4e893b2219dc] hfiles 2023-10-02T12:25:40,595 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/9f3d5d24e61b47dca538b66fb59a927a for snapshot=snapshot_backup_system 2023-10-02T12:25:40,595 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/3ba88e6f8052440dbf83bfb0e7a50568 for snapshot=snapshot_backup_system 2023-10-02T12:25:40,595 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/06206d275c734da89d8139ad46731eb7 for snapshot=snapshot_backup_system 2023-10-02T12:25:40,595 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/b7119e7f561746f787395f659b602c5b for snapshot=snapshot_backup_system 2023-10-02T12:25:40,595 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/2efcd96b58334c92986802f8ecd44bea for snapshot=snapshot_backup_system 2023-10-02T12:25:40,595 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/a6a3e92419324310b9bf4e893b2219dc for snapshot=snapshot_backup_system 2023-10-02T12:25:41,004 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:41,005 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=66}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=66 2023-10-02T12:25:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.HMaster(3995): Remote procedure done, pid=66 2023-10-02T12:25:41,005 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:41,007 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=66, ppid=65, state=RUNNABLE; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:41,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=66, resume processing ppid=65 2023-10-02T12:25:41,010 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:25:41,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=66, ppid=65, state=SUCCESS; SnapshotRegionProcedure 3dfc58254ae9ba5ba94c6dcd551aba30 in 1.4210 sec 2023-10-02T12:25:41,011 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:25:41,011 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:25:41,012 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:25:41,012 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:25:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=65 2023-10-02T12:25:41,426 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:25:41,440 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:25:41,441 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:25:41,442 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=65, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:25:41,442 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 65 2023-10-02T12:25:41,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=65, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=65, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2770 sec 2023-10-02T12:25:43,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=65 2023-10-02T12:25:43,287 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupAdminImpl(227): Deleting backup backup_1696249529969 ... 2023-10-02T12:25:43,289 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] util.BackupUtils(435): Delete log files: .backup.manifest 2023-10-02T12:25:43,291 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] util.BackupUtils(435): Delete log files: default 2023-10-02T12:25:43,292 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] util.BackupUtils(443): Trying to cleanup up target dir : backup_1696249529969 2023-10-02T12:25:43,292 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] util.BackupUtils(458): No data has been found in hdfs://localhost:36497/backupUT/backup_1696249529969/default/test-1696249492588. 2023-10-02T12:25:43,293 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] util.BackupUtils(465): hdfs://localhost:36497/backupUT/backup_1696249529969/default is empty, remove it. 2023-10-02T12:25:43,299 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupAdminImpl(269): 0 bulk loaded files out of 0 were deleted 2023-10-02T12:25:43,299 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(453): deleted 0 original bulkload rows 2023-10-02T12:25:43,301 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupAdminImpl(276): Delete backup backup_1696249529969 completed. 2023-10-02T12:25:43,307 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1034): test-1696249492588 2023-10-02T12:25:43,310 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:25:43,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:25:43,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:25:43,324 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:25:43,324 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(657): Finish backup exclusive operation DELETE operation finished OK: backup_1696249529969 No failed backup MERGE operation found 2023-10-02T12:25:43,327 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:25:43,337 ERROR [Listener at jenkins-hbase3.apache.org/37677 {}] impl.BackupSystemTable(1579): Snapshot snapshot_backup_system does not exists 2023-10-02T12:25:43,339 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:43,339 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:43,367 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] hbase.ResourceChecker(175): after: backup.TestBackupDeleteWithFailures#testBackupDeleteWithFailures Thread=596 (was 583) Potentially hanging thread: PacketResponder: BP-1592262056-172.31.12.81-1696249474750:blk_1073741905_1081, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:58172@0x4ab959a0-SendThread(127.0.0.1:58172) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-117100161_16 at /127.0.0.1:37192 [Receiving block BP-1592262056-172.31.12.81-1696249474750:blk_1073741905_1081] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,35033,1696249482294)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_890836690_16 at /127.0.0.1:60780 [Waiting for operation #12] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,35033,1696249482294' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-1592262056-172.31.12.81-1696249474750:blk_1073741906_1082, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/cluster_7356682d-271e-efa6-01f2-4b2f0febdaf5/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,35033,1696249482294)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1901042280_16 at /127.0.0.1:37264 [Waiting for operation #14] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:58172@0x1605f7a8-SendThread(127.0.0.1:58172) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:228) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1901042280_16 at /127.0.0.1:37250 [Waiting for operation #21] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,35033,1696249482294)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/cluster_7356682d-271e-efa6-01f2-4b2f0febdaf5/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:58172@0x1605f7a8 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$482/1013992421.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:58172@0x4ab959a0 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$482/1013992421.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:58172@0x4ab959a0-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0xed8f3b0-shared-pool-28 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,43611,1696249478718)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:58172@0x1605f7a8-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-117100161_16 at /127.0.0.1:37204 [Receiving block BP-1592262056-172.31.12.81-1696249474750:blk_1073741906_1082] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=690 (was 660) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=430 (was 522), ProcessCount=183 (was 183), AvailableMemoryMB=2539 (was 2496) - AvailableMemoryMB LEAK? - 2023-10-02T12:25:43,368 WARN [Listener at jenkins-hbase3.apache.org/37677 {}] hbase.ResourceChecker(130): Thread=596 is superior to 500 2023-10-02T12:25:43,378 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "hbase-failsafe-snapshot_backup_system-1696249512751" 2023-10-02T12:25:43,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43611 {}] snapshot.SnapshotManager(380): Deleting snapshot: hbase-failsafe-snapshot_backup_system-1696249512751 2023-10-02T12:25:43,383 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-10-02T12:25:43,384 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:43,384 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-10-02T12:25:43,384 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] util.JVMClusterUtil(257): Found active master hash=1522458711, stopped=false 2023-10-02T12:25:43,384 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.TestBackupDeleteWithFailures$MasterSnapshotObserver 2023-10-02T12:25:43,384 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,43611,1696249478718 2023-10-02T12:25:43,387 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] procedure2.ProcedureExecutor(630): Stopping 2023-10-02T12:25:43,387 DEBUG [Listener at jenkins-hbase3.apache.org/37677 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:43,387 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,35033,1696249482294' ***** 2023-10-02T12:25:43,387 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-10-02T12:25:43,387 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-10-02T12:25:43,404 INFO [RS:0;jenkins-hbase3:35033 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@584c4016{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-10-02T12:25:43,408 INFO [RS:0;jenkins-hbase3:35033 {}] server.AbstractConnector(383): Stopped ServerConnector@319cc17a{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:25:43,408 INFO [RS:0;jenkins-hbase3:35033 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-10-02T12:25:43,409 INFO [RS:0;jenkins-hbase3:35033 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@61caa118{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-10-02T12:25:43,410 INFO [RS:0;jenkins-hbase3:35033 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@4abd68ea{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/hadoop.log.dir/,STOPPED} 2023-10-02T12:25:43,413 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.HeapMemoryManager(220): Stopping 2023-10-02T12:25:43,413 INFO [RS:0;jenkins-hbase3:35033 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-10-02T12:25:43,414 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-10-02T12:25:43,414 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-10-02T12:25:43,414 INFO [RS:0;jenkins-hbase3:35033 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-10-02T12:25:43,414 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(3528): Received CLOSE for 8e88a763331d94795997dc16a6a7ee9b 2023-10-02T12:25:43,415 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(3528): Received CLOSE for 9ce225be3bc8c4e3b577db51ccdc0ec7 2023-10-02T12:25:43,415 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(3528): Received CLOSE for 0321d0b0338f45d0ba3fea553dbc434d 2023-10-02T12:25:43,415 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(3528): Received CLOSE for 3dfc58254ae9ba5ba94c6dcd551aba30 2023-10-02T12:25:43,415 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(3528): Received CLOSE for ff520da84fd90f4138d4152604b554a1 2023-10-02T12:25:43,415 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(3528): Received CLOSE for a09bef7b244cd13ac616d106a9188033 2023-10-02T12:25:43,415 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(3528): Received CLOSE for a39f2861eedc2f7c63f0bcd60085007e 2023-10-02T12:25:43,415 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,35033,1696249482294 2023-10-02T12:25:43,415 DEBUG [RS:0;jenkins-hbase3:35033 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:43,416 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 8e88a763331d94795997dc16a6a7ee9b, disabling compactions & flushes 2023-10-02T12:25:43,416 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b. 2023-10-02T12:25:43,416 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-10-02T12:25:43,416 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b. 2023-10-02T12:25:43,416 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-10-02T12:25:43,416 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-10-02T12:25:43,416 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b. after waiting 0 ms 2023-10-02T12:25:43,417 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b. 2023-10-02T12:25:43,417 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-10-02T12:25:43,417 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 8e88a763331d94795997dc16a6a7ee9b 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-10-02T12:25:43,418 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-10-02T12:25:43,418 DEBUG [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1583): Online Regions={8e88a763331d94795997dc16a6a7ee9b=hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b., 9ce225be3bc8c4e3b577db51ccdc0ec7=ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7., 0321d0b0338f45d0ba3fea553dbc434d=backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d., 3dfc58254ae9ba5ba94c6dcd551aba30=backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30., ff520da84fd90f4138d4152604b554a1=test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1., a09bef7b244cd13ac616d106a9188033=ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033., 1588230740=hbase:meta,,1.1588230740, a39f2861eedc2f7c63f0bcd60085007e=ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e.} 2023-10-02T12:25:43,419 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-10-02T12:25:43,419 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-10-02T12:25:43,419 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-10-02T12:25:43,419 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-10-02T12:25:43,419 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-10-02T12:25:43,419 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=13.22 KB heapSize=23.11 KB 2023-10-02T12:25:43,419 DEBUG [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1609): Waiting on 0321d0b0338f45d0ba3fea553dbc434d, 1588230740, 3dfc58254ae9ba5ba94c6dcd551aba30, 8e88a763331d94795997dc16a6a7ee9b, 9ce225be3bc8c4e3b577db51ccdc0ec7, a09bef7b244cd13ac616d106a9188033, a39f2861eedc2f7c63f0bcd60085007e, ff520da84fd90f4138d4152604b554a1 2023-10-02T12:25:43,424 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-10-02T12:25:43,424 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-10-02T12:25:43,451 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/namespace/8e88a763331d94795997dc16a6a7ee9b/.tmp/info/bd247a6247f241648a70de868186a8c7 is 45, key is default/info:d/1696249486717/Put/seqid=0 2023-10-02T12:25:43,454 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-10-02T12:25:43,456 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/meta/1588230740/.tmp/info/18f6df8b7cb34ba184c92bc8f06905dd is 159, key is ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e./info:regioninfo/1696249495569/Put/seqid=0 2023-10-02T12:25:43,467 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/namespace/8e88a763331d94795997dc16a6a7ee9b/.tmp/info/bd247a6247f241648a70de868186a8c7 2023-10-02T12:25:43,470 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.18 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/meta/1588230740/.tmp/info/18f6df8b7cb34ba184c92bc8f06905dd 2023-10-02T12:25:43,475 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/namespace/8e88a763331d94795997dc16a6a7ee9b/.tmp/info/bd247a6247f241648a70de868186a8c7 as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/namespace/8e88a763331d94795997dc16a6a7ee9b/info/bd247a6247f241648a70de868186a8c7 2023-10-02T12:25:43,481 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/namespace/8e88a763331d94795997dc16a6a7ee9b/info/bd247a6247f241648a70de868186a8c7, entries=7, sequenceid=11, filesize=5.1 K 2023-10-02T12:25:43,482 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for 8e88a763331d94795997dc16a6a7ee9b in 65ms, sequenceid=11, compaction requested=false 2023-10-02T12:25:43,492 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/namespace/8e88a763331d94795997dc16a6a7ee9b/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-10-02T12:25:43,493 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:25:43,494 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b. 2023-10-02T12:25:43,494 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 8e88a763331d94795997dc16a6a7ee9b: 2023-10-02T12:25:43,494 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1696249486021.8e88a763331d94795997dc16a6a7ee9b. 2023-10-02T12:25:43,495 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 9ce225be3bc8c4e3b577db51ccdc0ec7, disabling compactions & flushes 2023-10-02T12:25:43,495 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7. 2023-10-02T12:25:43,495 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7. 2023-10-02T12:25:43,495 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7. after waiting 0 ms 2023-10-02T12:25:43,495 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7. 2023-10-02T12:25:43,502 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/meta/1588230740/.tmp/table/faff5c1f81f14a2d9beb6165aa0ce116 is 59, key is ns2:test-16962494925881/table:state/1696249495580/Put/seqid=0 2023-10-02T12:25:43,503 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/ns4/test-16962494925883/9ce225be3bc8c4e3b577db51ccdc0ec7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-10-02T12:25:43,504 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:25:43,505 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7. 2023-10-02T12:25:43,505 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 9ce225be3bc8c4e3b577db51ccdc0ec7: 2023-10-02T12:25:43,505 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16962494925883,,1696249497222.9ce225be3bc8c4e3b577db51ccdc0ec7. 2023-10-02T12:25:43,506 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 0321d0b0338f45d0ba3fea553dbc434d, disabling compactions & flushes 2023-10-02T12:25:43,506 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d. 2023-10-02T12:25:43,507 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d. 2023-10-02T12:25:43,507 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d. after waiting 0 ms 2023-10-02T12:25:43,507 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d. 2023-10-02T12:25:43,509 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/meta/1588230740/.tmp/table/faff5c1f81f14a2d9beb6165aa0ce116 2023-10-02T12:25:43,512 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system_bulk/0321d0b0338f45d0ba3fea553dbc434d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-10-02T12:25:43,512 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:25:43,513 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d. 2023-10-02T12:25:43,513 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 0321d0b0338f45d0ba3fea553dbc434d: 2023-10-02T12:25:43,513 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1696249501278.0321d0b0338f45d0ba3fea553dbc434d. 2023-10-02T12:25:43,513 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 3dfc58254ae9ba5ba94c6dcd551aba30, disabling compactions & flushes 2023-10-02T12:25:43,513 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:43,513 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:43,513 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. after waiting 0 ms 2023-10-02T12:25:43,514 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:43,514 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 3dfc58254ae9ba5ba94c6dcd551aba30 2/2 column families, dataSize=223 B heapSize=1.01 KB 2023-10-02T12:25:43,517 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/meta/1588230740/.tmp/info/18f6df8b7cb34ba184c92bc8f06905dd as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/meta/1588230740/info/18f6df8b7cb34ba184c92bc8f06905dd 2023-10-02T12:25:43,520 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/0de02cd116854d6bbe8440850e77d25a is 91, key is incrbackupset:hdfs://localhost:36497/backupUT/meta:test-1696249492588/1696249543308/Put/seqid=0 2023-10-02T12:25:43,523 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/meta/1588230740/info/18f6df8b7cb34ba184c92bc8f06905dd, entries=72, sequenceid=59, filesize=14.3 K 2023-10-02T12:25:43,524 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/meta/1588230740/.tmp/table/faff5c1f81f14a2d9beb6165aa0ce116 as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/meta/1588230740/table/faff5c1f81f14a2d9beb6165aa0ce116 2023-10-02T12:25:43,531 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/meta/1588230740/table/faff5c1f81f14a2d9beb6165aa0ce116, entries=15, sequenceid=59, filesize=5.8 K 2023-10-02T12:25:43,531 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~13.22 KB/13542, heapSize ~22.83 KB/23376, currentSize=0 B/0 for 1588230740 in 112ms, sequenceid=59, compaction requested=false 2023-10-02T12:25:43,541 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/hbase/meta/1588230740/recovered.edits/62.seqid, newMaxSeqId=62, maxSeqId=1 2023-10-02T12:25:43,542 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:25:43,542 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-02T12:25:43,542 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-10-02T12:25:43,542 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-10-02T12:25:43,542 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-10-02T12:25:43,620 DEBUG [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1609): Waiting on 3dfc58254ae9ba5ba94c6dcd551aba30, a09bef7b244cd13ac616d106a9188033, a39f2861eedc2f7c63f0bcd60085007e, ff520da84fd90f4138d4152604b554a1 2023-10-02T12:25:43,820 DEBUG [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1609): Waiting on 3dfc58254ae9ba5ba94c6dcd551aba30, a09bef7b244cd13ac616d106a9188033, a39f2861eedc2f7c63f0bcd60085007e, ff520da84fd90f4138d4152604b554a1 2023-10-02T12:25:43,930 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=124 B at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/0de02cd116854d6bbe8440850e77d25a 2023-10-02T12:25:43,936 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0de02cd116854d6bbe8440850e77d25a 2023-10-02T12:25:43,941 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/a23314f8845b4ed098b2e2a2b41b60d2 is 59, key is session:backup_1696249529969/session:/1696249543300/DeleteFamily/seqid=0 2023-10-02T12:25:44,020 DEBUG [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1609): Waiting on 3dfc58254ae9ba5ba94c6dcd551aba30, a09bef7b244cd13ac616d106a9188033, a39f2861eedc2f7c63f0bcd60085007e, ff520da84fd90f4138d4152604b554a1 2023-10-02T12:25:44,220 DEBUG [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1609): Waiting on 3dfc58254ae9ba5ba94c6dcd551aba30, a09bef7b244cd13ac616d106a9188033, a39f2861eedc2f7c63f0bcd60085007e, ff520da84fd90f4138d4152604b554a1 2023-10-02T12:25:44,346 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=99 B at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/a23314f8845b4ed098b2e2a2b41b60d2 2023-10-02T12:25:44,359 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a23314f8845b4ed098b2e2a2b41b60d2 2023-10-02T12:25:44,367 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/meta/0de02cd116854d6bbe8440850e77d25a as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/0de02cd116854d6bbe8440850e77d25a 2023-10-02T12:25:44,374 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0de02cd116854d6bbe8440850e77d25a 2023-10-02T12:25:44,374 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/meta/0de02cd116854d6bbe8440850e77d25a, entries=2, sequenceid=87, filesize=5.2 K 2023-10-02T12:25:44,375 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/.tmp/session/a23314f8845b4ed098b2e2a2b41b60d2 as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/a23314f8845b4ed098b2e2a2b41b60d2 2023-10-02T12:25:44,377 DEBUG [regionserver/jenkins-hbase3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.23076923076923078, tune throughput to 61.54 MB/second 2023-10-02T12:25:44,381 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a23314f8845b4ed098b2e2a2b41b60d2 2023-10-02T12:25:44,381 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/session/a23314f8845b4ed098b2e2a2b41b60d2, entries=2, sequenceid=87, filesize=5.2 K 2023-10-02T12:25:44,382 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~223 B/223, heapSize ~1000 B/1000, currentSize=0 B/0 for 3dfc58254ae9ba5ba94c6dcd551aba30 in 868ms, sequenceid=87, compaction requested=true 2023-10-02T12:25:44,421 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1579): Waiting on 4 regions to close 2023-10-02T12:25:44,421 DEBUG [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1583): Online Regions={3dfc58254ae9ba5ba94c6dcd551aba30=backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30., ff520da84fd90f4138d4152604b554a1=test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1., a09bef7b244cd13ac616d106a9188033=ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033., a39f2861eedc2f7c63f0bcd60085007e=ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e.} 2023-10-02T12:25:44,421 DEBUG [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1609): Waiting on 3dfc58254ae9ba5ba94c6dcd551aba30, a09bef7b244cd13ac616d106a9188033, a39f2861eedc2f7c63f0bcd60085007e, ff520da84fd90f4138d4152604b554a1 2023-10-02T12:25:44,424 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/backup/system/3dfc58254ae9ba5ba94c6dcd551aba30/recovered.edits/90.seqid, newMaxSeqId=90, maxSeqId=34 2023-10-02T12:25:44,425 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:25:44,426 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:44,426 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 3dfc58254ae9ba5ba94c6dcd551aba30: 2023-10-02T12:25:44,426 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1696249499136.3dfc58254ae9ba5ba94c6dcd551aba30. 2023-10-02T12:25:44,426 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing ff520da84fd90f4138d4152604b554a1, disabling compactions & flushes 2023-10-02T12:25:44,426 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. 2023-10-02T12:25:44,426 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. 2023-10-02T12:25:44,426 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. after waiting 0 ms 2023-10-02T12:25:44,426 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. 2023-10-02T12:25:44,431 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/default/test-1696249492588/ff520da84fd90f4138d4152604b554a1/recovered.edits/108.seqid, newMaxSeqId=108, maxSeqId=1 2023-10-02T12:25:44,432 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:25:44,433 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. 2023-10-02T12:25:44,433 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for ff520da84fd90f4138d4152604b554a1: 2023-10-02T12:25:44,433 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1696249492588,,1696249493086.ff520da84fd90f4138d4152604b554a1. 2023-10-02T12:25:44,434 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing a09bef7b244cd13ac616d106a9188033, disabling compactions & flushes 2023-10-02T12:25:44,434 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033. 2023-10-02T12:25:44,434 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033. 2023-10-02T12:25:44,434 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033. after waiting 0 ms 2023-10-02T12:25:44,434 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033. 2023-10-02T12:25:44,438 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/ns3/test-16962494925882/a09bef7b244cd13ac616d106a9188033/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-10-02T12:25:44,438 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:25:44,439 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033. 2023-10-02T12:25:44,439 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for a09bef7b244cd13ac616d106a9188033: 2023-10-02T12:25:44,439 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16962494925882,,1696249496596.a09bef7b244cd13ac616d106a9188033. 2023-10-02T12:25:44,440 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing a39f2861eedc2f7c63f0bcd60085007e, disabling compactions & flushes 2023-10-02T12:25:44,440 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e. 2023-10-02T12:25:44,440 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e. 2023-10-02T12:25:44,440 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e. after waiting 0 ms 2023-10-02T12:25:44,440 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e. 2023-10-02T12:25:44,440 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing a39f2861eedc2f7c63f0bcd60085007e 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-10-02T12:25:44,456 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/ns2/test-16962494925881/a39f2861eedc2f7c63f0bcd60085007e/.tmp/f/00674d10becb4305abf17819447bb442 is 37, key is row10/f:q1/1696249496489/Put/seqid=0 2023-10-02T12:25:44,621 DEBUG [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1609): Waiting on a39f2861eedc2f7c63f0bcd60085007e 2023-10-02T12:25:44,822 DEBUG [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1609): Waiting on a39f2861eedc2f7c63f0bcd60085007e 2023-10-02T12:25:44,861 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/ns2/test-16962494925881/a39f2861eedc2f7c63f0bcd60085007e/.tmp/f/00674d10becb4305abf17819447bb442 2023-10-02T12:25:44,871 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/ns2/test-16962494925881/a39f2861eedc2f7c63f0bcd60085007e/.tmp/f/00674d10becb4305abf17819447bb442 as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/ns2/test-16962494925881/a39f2861eedc2f7c63f0bcd60085007e/f/00674d10becb4305abf17819447bb442 2023-10-02T12:25:44,880 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/ns2/test-16962494925881/a39f2861eedc2f7c63f0bcd60085007e/f/00674d10becb4305abf17819447bb442, entries=99, sequenceid=103, filesize=8.2 K 2023-10-02T12:25:44,881 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for a39f2861eedc2f7c63f0bcd60085007e in 441ms, sequenceid=103, compaction requested=false 2023-10-02T12:25:44,926 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/data/ns2/test-16962494925881/a39f2861eedc2f7c63f0bcd60085007e/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-10-02T12:25:44,927 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:25:44,928 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e. 2023-10-02T12:25:44,928 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for a39f2861eedc2f7c63f0bcd60085007e: 2023-10-02T12:25:44,928 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16962494925881,,1696249494352.a39f2861eedc2f7c63f0bcd60085007e. 2023-10-02T12:25:45,022 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,35033,1696249482294; all regions closed. 2023-10-02T12:25:45,032 DEBUG [RS:0;jenkins-hbase3:35033 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/oldWALs 2023-10-02T12:25:45,032 INFO [RS:0;jenkins-hbase3:35033 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C35033%2C1696249482294.meta:.meta(num 1696249534173) 2023-10-02T12:25:45,041 DEBUG [RS:0;jenkins-hbase3:35033 {}] wal.AbstractFSWAL(1062): Moved 3 WAL file(s) to /user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/oldWALs 2023-10-02T12:25:45,041 INFO [RS:0;jenkins-hbase3:35033 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C35033%2C1696249482294:(num 1696249534147) 2023-10-02T12:25:45,041 DEBUG [RS:0;jenkins-hbase3:35033 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:45,041 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.LeaseManager(133): Closed leases 2023-10-02T12:25:45,041 INFO [RS:0;jenkins-hbase3:35033 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2023-10-02T12:25:45,042 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-10-02T12:25:45,043 INFO [RS:0;jenkins-hbase3:35033 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:35033 2023-10-02T12:25:45,053 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,35033,1696249482294] 2023-10-02T12:25:45,053 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,35033,1696249482294; numProcessing=1 2023-10-02T12:25:45,055 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,35033,1696249482294 expired; onlineServers=0 2023-10-02T12:25:45,055 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,43611,1696249478718' ***** 2023-10-02T12:25:45,055 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-10-02T12:25:45,056 DEBUG [M:0;jenkins-hbase3:43611 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388df16b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-10-02T12:25:45,056 INFO [M:0;jenkins-hbase3:43611 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-10-02T12:25:45,107 INFO [M:0;jenkins-hbase3:43611 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@3e051dc7{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-10-02T12:25:45,108 INFO [M:0;jenkins-hbase3:43611 {}] server.AbstractConnector(383): Stopped ServerConnector@33782983{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:25:45,108 INFO [M:0;jenkins-hbase3:43611 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-10-02T12:25:45,109 INFO [M:0;jenkins-hbase3:43611 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5f5c5c02{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-10-02T12:25:45,109 INFO [M:0;jenkins-hbase3:43611 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@6a56522d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/hadoop.log.dir/,STOPPED} 2023-10-02T12:25:45,109 INFO [M:0;jenkins-hbase3:43611 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,43611,1696249478718 2023-10-02T12:25:45,110 INFO [M:0;jenkins-hbase3:43611 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,43611,1696249478718; all regions closed. 2023-10-02T12:25:45,110 DEBUG [M:0;jenkins-hbase3:43611 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:45,110 INFO [M:0;jenkins-hbase3:43611 {}] master.HMaster(1665): Stopping master jetty server 2023-10-02T12:25:45,110 INFO [M:0;jenkins-hbase3:43611 {}] server.AbstractConnector(383): Stopped ServerConnector@58bcb309{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:25:45,111 DEBUG [M:0;jenkins-hbase3:43611 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-10-02T12:25:45,111 INFO [M:0;jenkins-hbase3:43611 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-10-02T12:25:45,111 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-10-02T12:25:45,111 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696249484270 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696249484270,5,FailOnTimeoutGroup] 2023-10-02T12:25:45,111 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696249484274 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696249484274,5,FailOnTimeoutGroup] 2023-10-02T12:25:45,111 DEBUG [M:0;jenkins-hbase3:43611 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:25:45,111 DEBUG [M:0;jenkins-hbase3:43611 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-10-02T12:25:45,111 INFO [M:0;jenkins-hbase3:43611 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-10-02T12:25:45,112 DEBUG [M:0;jenkins-hbase3:43611 {}] master.HMaster(1688): Stopping service threads 2023-10-02T12:25:45,112 INFO [M:0;jenkins-hbase3:43611 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-10-02T12:25:45,112 ERROR [M:0;jenkins-hbase3:43611 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] 2023-10-02T12:25:45,112 INFO [M:0;jenkins-hbase3:43611 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-10-02T12:25:45,112 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-10-02T12:25:45,113 WARN [M:0;jenkins-hbase3:43611 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-10-02T12:25:45,113 INFO [M:0;jenkins-hbase3:43611 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-10-02T12:25:45,113 INFO [M:0;jenkins-hbase3:43611 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-10-02T12:25:45,114 DEBUG [M:0;jenkins-hbase3:43611 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-10-02T12:25:45,125 INFO [M:0;jenkins-hbase3:43611 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:25:45,125 DEBUG [M:0;jenkins-hbase3:43611 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:25:45,125 DEBUG [M:0;jenkins-hbase3:43611 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-10-02T12:25:45,125 DEBUG [M:0;jenkins-hbase3:43611 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:25:45,125 INFO [M:0;jenkins-hbase3:43611 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=254.27 KB heapSize=305.33 KB 2023-10-02T12:25:45,139 DEBUG [M:0;jenkins-hbase3:43611 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/80ff5368e1d94657a78f0be4404aed8d is 88, key is hbase:meta,,1/info:sn/1696249485289/Put/seqid=0 2023-10-02T12:25:45,153 INFO [RS:0;jenkins-hbase3:35033 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,35033,1696249482294; zookeeper connection closed. 2023-10-02T12:25:45,154 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@665cb050 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@665cb050 2023-10-02T12:25:45,154 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-10-02T12:25:45,543 INFO [M:0;jenkins-hbase3:43611 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/80ff5368e1d94657a78f0be4404aed8d 2023-10-02T12:25:45,565 DEBUG [M:0;jenkins-hbase3:43611 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e49b31bbfcda4511868a718eccb39d19 is 2093, key is \x00\x00\x00\x00\x00\x00\x00@/proc:d/1696249538940/Put/seqid=0 2023-10-02T12:25:45,975 INFO [M:0;jenkins-hbase3:43611 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=253.67 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e49b31bbfcda4511868a718eccb39d19 2023-10-02T12:25:45,981 INFO [M:0;jenkins-hbase3:43611 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e49b31bbfcda4511868a718eccb39d19 2023-10-02T12:25:45,994 DEBUG [M:0;jenkins-hbase3:43611 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1ddf311651a349c9a16294ef8550f355 is 82, key is jenkins-hbase3.apache.org,35033,1696249482294/rs:state/1696249484293/Put/seqid=0 2023-10-02T12:25:46,399 INFO [M:0;jenkins-hbase3:43611 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1ddf311651a349c9a16294ef8550f355 2023-10-02T12:25:46,406 DEBUG [M:0;jenkins-hbase3:43611 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/80ff5368e1d94657a78f0be4404aed8d as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/80ff5368e1d94657a78f0be4404aed8d 2023-10-02T12:25:46,411 INFO [M:0;jenkins-hbase3:43611 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/80ff5368e1d94657a78f0be4404aed8d, entries=8, sequenceid=606, filesize=5.5 K 2023-10-02T12:25:46,412 DEBUG [M:0;jenkins-hbase3:43611 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e49b31bbfcda4511868a718eccb39d19 as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e49b31bbfcda4511868a718eccb39d19 2023-10-02T12:25:46,417 INFO [M:0;jenkins-hbase3:43611 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e49b31bbfcda4511868a718eccb39d19 2023-10-02T12:25:46,417 INFO [M:0;jenkins-hbase3:43611 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e49b31bbfcda4511868a718eccb39d19, entries=66, sequenceid=606, filesize=22.1 K 2023-10-02T12:25:46,418 DEBUG [M:0;jenkins-hbase3:43611 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1ddf311651a349c9a16294ef8550f355 as hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1ddf311651a349c9a16294ef8550f355 2023-10-02T12:25:46,423 INFO [M:0;jenkins-hbase3:43611 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:36497/user/jenkins/test-data/5a44280e-cfc6-1ea9-4bb3-e74f6939935a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1ddf311651a349c9a16294ef8550f355, entries=1, sequenceid=606, filesize=5.1 K 2023-10-02T12:25:46,423 INFO [M:0;jenkins-hbase3:43611 {}] regionserver.HRegion(3022): Finished flush of dataSize ~254.27 KB/260370, heapSize ~305.03 KB/312352, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1298ms, sequenceid=606, compaction requested=false 2023-10-02T12:25:46,428 INFO [M:0;jenkins-hbase3:43611 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:25:46,428 DEBUG [M:0;jenkins-hbase3:43611 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:25:46,431 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-10-02T12:25:46,431 INFO [M:0;jenkins-hbase3:43611 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-10-02T12:25:46,431 INFO [M:0;jenkins-hbase3:43611 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-10-02T12:25:46,431 INFO [M:0;jenkins-hbase3:43611 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:43611 2023-10-02T12:25:46,535 INFO [M:0;jenkins-hbase3:43611 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,43611,1696249478718; zookeeper connection closed. 2023-10-02T12:25:46,537 WARN [Listener at jenkins-hbase3.apache.org/37677 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-10-02T12:25:46,544 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-10-02T12:25:46,653 WARN [BP-1592262056-172.31.12.81-1696249474750 heartbeating to localhost/127.0.0.1:36497 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-10-02T12:25:46,653 WARN [BP-1592262056-172.31.12.81-1696249474750 heartbeating to localhost/127.0.0.1:36497 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-1592262056-172.31.12.81-1696249474750 (Datanode Uuid 2484e56c-fb12-4c55-8990-9f71d5262c43) service to localhost/127.0.0.1:36497 2023-10-02T12:25:46,655 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/cluster_7356682d-271e-efa6-01f2-4b2f0febdaf5/dfs/data/data1/current/BP-1592262056-172.31.12.81-1696249474750 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-02T12:25:46,656 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/a451c2f0-757d-8905-eb2c-3d9516208da0/cluster_7356682d-271e-efa6-01f2-4b2f0febdaf5/dfs/data/data2/current/BP-1592262056-172.31.12.81-1696249474750 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-02T12:25:46,696 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-10-02T12:25:46,878 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-10-02T12:25:46,878 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-10-02T12:25:46,893 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:25:47,013 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:25:47,125 ERROR [Thread[Thread-275,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:25:47,126 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:25:47,241 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-10-02T12:25:47,249 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-10-02T12:25:47,250 ERROR [Thread[Thread-286,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:25:47,257 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:25:47,360 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:25:47,360 INFO [Listener at jenkins-hbase3.apache.org/37677 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped