2023-10-02 12:29:33,254 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 2023-10-02 12:29:33,267 main DEBUG Took 0.010850 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-10-02 12:29:33,267 main DEBUG PluginManager 'Core' found 129 plugins 2023-10-02 12:29:33,267 main DEBUG PluginManager 'Level' found 0 plugins 2023-10-02 12:29:33,268 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-10-02 12:29:33,270 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:29:33,279 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-10-02 12:29:33,295 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:29:33,296 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:29:33,297 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:29:33,297 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:29:33,297 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:29:33,298 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:29:33,298 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:29:33,299 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:29:33,299 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:29:33,299 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:29:33,300 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:29:33,300 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:29:33,301 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:29:33,301 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:29:33,301 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:29:33,302 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:29:33,302 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:29:33,302 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:29:33,303 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:29:33,303 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:29:33,304 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:29:33,304 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:29:33,305 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:29:33,305 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-10-02 12:29:33,306 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:29:33,306 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-10-02 12:29:33,309 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-10-02 12:29:33,311 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-10-02 12:29:33,313 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-10-02 12:29:33,314 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-10-02 12:29:33,315 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-10-02 12:29:33,315 main DEBUG PluginManager 'Converter' found 47 plugins 2023-10-02 12:29:33,327 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-10-02 12:29:33,329 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-10-02 12:29:33,334 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-10-02 12:29:33,334 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-10-02 12:29:33,334 main DEBUG createAppenders(={Console}) 2023-10-02 12:29:33,335 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 initialized 2023-10-02 12:29:33,335 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 2023-10-02 12:29:33,336 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@175c2241 OK. 2023-10-02 12:29:33,336 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-10-02 12:29:33,337 main DEBUG OutputStream closed 2023-10-02 12:29:33,337 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-10-02 12:29:33,337 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-10-02 12:29:33,337 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@2145b572 OK 2023-10-02 12:29:33,386 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-10-02 12:29:33,388 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-10-02 12:29:33,389 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-10-02 12:29:33,390 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-10-02 12:29:33,391 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-10-02 12:29:33,391 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-10-02 12:29:33,391 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-10-02 12:29:33,392 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-10-02 12:29:33,392 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-10-02 12:29:33,392 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-10-02 12:29:33,392 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-10-02 12:29:33,393 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-10-02 12:29:33,393 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-10-02 12:29:33,393 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-10-02 12:29:33,394 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-10-02 12:29:33,394 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-10-02 12:29:33,394 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-10-02 12:29:33,395 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-10-02 12:29:33,398 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-10-02 12:29:33,398 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@49912c99) with optional ClassLoader: null 2023-10-02 12:29:33,399 main DEBUG Shutdown hook enabled. Registering a new one. 2023-10-02 12:29:33,400 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@49912c99] started OK. 2023-10-02T12:29:33,430 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestFullRestore timeout: 13 mins 2023-10-02 12:29:33,434 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-10-02 12:29:33,434 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-10-02T12:29:33,794 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7 2023-10-02T12:29:33,802 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-10-02T12:29:33,805 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:29:33,807 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-10-02T12:29:33,812 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/cluster_f5efbd98-923a-ea34-67ad-be7e4cbb4c4b, deleteOnExit=true 2023-10-02T12:29:33,813 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-10-02T12:29:33,813 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/test.cache.data in system properties and HBase conf 2023-10-02T12:29:33,814 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop.tmp.dir in system properties and HBase conf 2023-10-02T12:29:33,815 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop.log.dir in system properties and HBase conf 2023-10-02T12:29:33,815 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir in system properties and HBase conf 2023-10-02T12:29:33,816 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-10-02T12:29:33,816 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-10-02T12:29:33,957 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-10-02T12:29:34,516 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-10-02T12:29:34,523 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:29:34,523 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:29:34,524 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-10-02T12:29:34,524 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:29:34,524 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-10-02T12:29:34,525 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-10-02T12:29:34,525 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:29:34,525 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:29:34,526 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-10-02T12:29:34,526 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/nfs.dump.dir in system properties and HBase conf 2023-10-02T12:29:34,526 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/java.io.tmpdir in system properties and HBase conf 2023-10-02T12:29:34,526 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:29:34,527 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-10-02T12:29:34,527 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-10-02T12:29:35,069 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-10-02T12:29:35,073 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-10-02T12:29:35,378 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-10-02T12:29:35,537 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@2d56dd0b via org.mortbay.log.Slf4jLog 2023-10-02T12:29:35,566 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:29:35,622 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:29:35,675 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/java.io.tmpdir/Jetty_localhost_37061_hdfs____eyi3ld/webapp 2023-10-02T12:29:35,865 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37061 2023-10-02T12:29:35,902 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-10-02T12:29:35,902 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-10-02T12:29:36,587 WARN [Listener at localhost/37823 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:29:36,681 WARN [Listener at localhost/37823 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-10-02T12:29:36,707 WARN [Listener at localhost/37823 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:29:36,716 INFO [Listener at localhost/37823 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:29:36,726 INFO [Listener at localhost/37823 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/java.io.tmpdir/Jetty_localhost_45391_datanode____.ydqysv/webapp 2023-10-02T12:29:36,867 INFO [Listener at localhost/37823 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:45391 2023-10-02T12:29:37,214 WARN [Listener at localhost/45111 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-10-02T12:29:37,549 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-10-02T12:29:37,764 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x11c7c26a84fb14a0: Processing first storage report for DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7 from datanode e74fb602-d5df-4cc6-94ee-2a855013b8e8 2023-10-02T12:29:37,766 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x11c7c26a84fb14a0: from storage DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7 node DatanodeRegistration(127.0.0.1:34237, datanodeUuid=e74fb602-d5df-4cc6-94ee-2a855013b8e8, infoPort=45865, infoSecurePort=0, ipcPort=45111, storageInfo=lv=-57;cid=testClusterID;nsid=418260307;c=1696249775150), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2023-10-02T12:29:37,766 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x11c7c26a84fb14a0: Processing first storage report for DS-7039dc59-bf3b-4804-8595-3401ddf54488 from datanode e74fb602-d5df-4cc6-94ee-2a855013b8e8 2023-10-02T12:29:37,766 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x11c7c26a84fb14a0: from storage DS-7039dc59-bf3b-4804-8595-3401ddf54488 node DatanodeRegistration(127.0.0.1:34237, datanodeUuid=e74fb602-d5df-4cc6-94ee-2a855013b8e8, infoPort=45865, infoSecurePort=0, ipcPort=45111, storageInfo=lv=-57;cid=testClusterID;nsid=418260307;c=1696249775150), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-10-02T12:29:37,820 DEBUG [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7 2023-10-02T12:29:37,955 INFO [Listener at localhost/45111 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/cluster_f5efbd98-923a-ea34-67ad-be7e4cbb4c4b/zookeeper_0, clientPort=57768, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/cluster_f5efbd98-923a-ea34-67ad-be7e4cbb4c4b/zookeeper_0/version-2, dataDirSize=424 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/cluster_f5efbd98-923a-ea34-67ad-be7e4cbb4c4b/zookeeper_0/version-2, dataLogSize=424 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, serverId=0 2023-10-02T12:29:37,979 INFO [Listener at localhost/45111 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=57768 2023-10-02T12:29:37,989 INFO [Listener at localhost/45111 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:29:37,992 INFO [Listener at localhost/45111 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:29:38,722 INFO [Listener at localhost/45111 {}] util.FSUtils(462): Created version file at hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 with version=8 2023-10-02T12:29:38,723 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging 2023-10-02T12:29:38,746 DEBUG [Listener at localhost/45111 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-10-02T12:29:38,746 DEBUG [Listener at localhost/45111 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-10-02T12:29:38,746 DEBUG [Listener at localhost/45111 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-10-02T12:29:38,746 DEBUG [Listener at localhost/45111 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-10-02T12:29:38,985 DEBUG [Listener at localhost/45111 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-10-02T12:29:39,808 INFO [Listener at localhost/45111 {}] client.ConnectionUtils(128): master/jenkins-hbase3:0 server-side Connection retries=45 2023-10-02T12:29:39,855 INFO [Listener at localhost/45111 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:29:39,855 INFO [Listener at localhost/45111 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-10-02T12:29:39,856 INFO [Listener at localhost/45111 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-10-02T12:29:39,856 INFO [Listener at localhost/45111 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:29:39,856 INFO [Listener at localhost/45111 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-10-02T12:29:40,010 INFO [Listener at localhost/45111 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-10-02T12:29:40,139 INFO [Listener at localhost/45111 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-10-02T12:29:40,159 DEBUG [Listener at localhost/45111 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-10-02T12:29:40,166 INFO [Listener at localhost/45111 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-10-02T12:29:40,215 DEBUG [Listener at localhost/45111 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 9321 (auto-detected) 2023-10-02T12:29:40,217 DEBUG [Listener at localhost/45111 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:1c:ff:fe:e2:13:ea (auto-detected) 2023-10-02T12:29:40,254 INFO [Listener at localhost/45111 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:40619 2023-10-02T12:29:40,269 INFO [Listener at localhost/45111 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:29:40,271 INFO [Listener at localhost/45111 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:29:40,292 INFO [Listener at localhost/45111 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:40619 connecting to ZooKeeper ensemble=127.0.0.1:57768 2023-10-02T12:29:40,340 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:406190x0, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-10-02T12:29:40,344 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:40619-0x1017326ef4b0000 connected 2023-10-02T12:29:40,370 DEBUG [Listener at localhost/45111 {}] zookeeper.ZKUtil(113): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-10-02T12:29:40,374 DEBUG [Listener at localhost/45111 {}] zookeeper.ZKUtil(113): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-10-02T12:29:40,378 DEBUG [Listener at localhost/45111 {}] zookeeper.ZKUtil(113): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-10-02T12:29:40,388 DEBUG [Listener at localhost/45111 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40619 2023-10-02T12:29:40,389 DEBUG [Listener at localhost/45111 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40619 2023-10-02T12:29:40,390 DEBUG [Listener at localhost/45111 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40619 2023-10-02T12:29:40,390 DEBUG [Listener at localhost/45111 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40619 2023-10-02T12:29:40,390 DEBUG [Listener at localhost/45111 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40619 2023-10-02T12:29:40,444 INFO [Listener at localhost/45111 {}] log.Log(170): Logging initialized @7890ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-10-02T12:29:40,570 INFO [Listener at localhost/45111 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-10-02T12:29:40,571 INFO [Listener at localhost/45111 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-10-02T12:29:40,572 INFO [Listener at localhost/45111 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-10-02T12:29:40,574 INFO [Listener at localhost/45111 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-10-02T12:29:40,574 INFO [Listener at localhost/45111 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-10-02T12:29:40,574 INFO [Listener at localhost/45111 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-10-02T12:29:40,578 INFO [Listener at localhost/45111 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-10-02T12:29:40,724 INFO [Listener at localhost/45111 {}] http.HttpServer(1219): Jetty bound to port 42929 2023-10-02T12:29:40,727 INFO [Listener at localhost/45111 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:29:40,764 INFO [Listener at localhost/45111 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:29:40,768 INFO [Listener at localhost/45111 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@2cb278af{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop.log.dir/,AVAILABLE} 2023-10-02T12:29:40,769 INFO [Listener at localhost/45111 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:29:40,769 INFO [Listener at localhost/45111 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5dbdc559{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-10-02T12:29:40,834 INFO [Listener at localhost/45111 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-10-02T12:29:40,848 INFO [Listener at localhost/45111 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-10-02T12:29:40,848 INFO [Listener at localhost/45111 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-10-02T12:29:40,851 INFO [Listener at localhost/45111 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-10-02T12:29:40,858 INFO [Listener at localhost/45111 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:29:42,032 INFO [Listener at localhost/45111 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@5762bd3d{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-10-02T12:29:42,043 INFO [Listener at localhost/45111 {}] server.AbstractConnector(333): Started ServerConnector@35502a4a{HTTP/1.1, (http/1.1)}{0.0.0.0:42929} 2023-10-02T12:29:42,044 INFO [Listener at localhost/45111 {}] server.Server(415): Started @9489ms 2023-10-02T12:29:42,047 INFO [Listener at localhost/45111 {}] master.HMaster(485): hbase.rootdir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07, hbase.cluster.distributed=false 2023-10-02T12:29:42,115 INFO [Listener at localhost/45111 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase3:0 server-side Connection retries=45 2023-10-02T12:29:42,116 INFO [Listener at localhost/45111 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:29:42,116 INFO [Listener at localhost/45111 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-10-02T12:29:42,116 INFO [Listener at localhost/45111 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-10-02T12:29:42,116 INFO [Listener at localhost/45111 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-10-02T12:29:42,117 INFO [Listener at localhost/45111 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-10-02T12:29:42,127 INFO [Listener at localhost/45111 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-10-02T12:29:42,129 INFO [Listener at localhost/45111 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-10-02T12:29:42,130 INFO [Listener at localhost/45111 {}] ipc.NettyRpcServer(192): Bind to /172.31.12.81:38255 2023-10-02T12:29:42,133 INFO [Listener at localhost/45111 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-10-02T12:29:42,141 DEBUG [Listener at localhost/45111 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-10-02T12:29:42,142 INFO [Listener at localhost/45111 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:29:42,145 INFO [Listener at localhost/45111 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:29:42,148 INFO [Listener at localhost/45111 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:38255 connecting to ZooKeeper ensemble=127.0.0.1:57768 2023-10-02T12:29:42,151 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:382550x0, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-10-02T12:29:42,152 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:38255-0x1017326ef4b0001 connected 2023-10-02T12:29:42,153 DEBUG [Listener at localhost/45111 {}] zookeeper.ZKUtil(113): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-10-02T12:29:42,154 DEBUG [Listener at localhost/45111 {}] zookeeper.ZKUtil(113): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-10-02T12:29:42,155 DEBUG [Listener at localhost/45111 {}] zookeeper.ZKUtil(113): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-10-02T12:29:42,156 DEBUG [Listener at localhost/45111 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38255 2023-10-02T12:29:42,156 DEBUG [Listener at localhost/45111 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38255 2023-10-02T12:29:42,157 DEBUG [Listener at localhost/45111 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38255 2023-10-02T12:29:42,157 DEBUG [Listener at localhost/45111 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38255 2023-10-02T12:29:42,157 DEBUG [Listener at localhost/45111 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38255 2023-10-02T12:29:42,161 INFO [Listener at localhost/45111 {}] http.HttpServer(973): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-10-02T12:29:42,161 INFO [Listener at localhost/45111 {}] http.HttpServer(973): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-10-02T12:29:42,161 INFO [Listener at localhost/45111 {}] http.HttpServer(973): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-10-02T12:29:42,162 INFO [Listener at localhost/45111 {}] http.HttpServer(952): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-10-02T12:29:42,162 INFO [Listener at localhost/45111 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-10-02T12:29:42,163 INFO [Listener at localhost/45111 {}] http.HttpServer(959): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-10-02T12:29:42,163 INFO [Listener at localhost/45111 {}] http.HttpServer(797): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-10-02T12:29:42,165 INFO [Listener at localhost/45111 {}] http.HttpServer(1219): Jetty bound to port 34265 2023-10-02T12:29:42,165 INFO [Listener at localhost/45111 {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:29:42,167 INFO [Listener at localhost/45111 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:29:42,167 INFO [Listener at localhost/45111 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@71013728{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop.log.dir/,AVAILABLE} 2023-10-02T12:29:42,167 INFO [Listener at localhost/45111 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:29:42,168 INFO [Listener at localhost/45111 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@ae5dddd{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-10-02T12:29:42,184 INFO [Listener at localhost/45111 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-10-02T12:29:42,185 INFO [Listener at localhost/45111 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-10-02T12:29:42,186 INFO [Listener at localhost/45111 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-10-02T12:29:42,186 INFO [Listener at localhost/45111 {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2023-10-02T12:29:42,187 INFO [Listener at localhost/45111 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-10-02T12:29:42,192 INFO [Listener at localhost/45111 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@1261a172{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-10-02T12:29:42,193 INFO [Listener at localhost/45111 {}] server.AbstractConnector(333): Started ServerConnector@5d09fa3f{HTTP/1.1, (http/1.1)}{0.0.0.0:34265} 2023-10-02T12:29:42,194 INFO [Listener at localhost/45111 {}] server.Server(415): Started @9639ms 2023-10-02T12:29:42,198 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.50.v20221201; built: 2022-12-01T22:07:03.915Z; git: da9a0b30691a45daf90a9f17b5defa2f1434f882; jvm 1.8.0_362-b09 2023-10-02T12:29:42,205 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@eab1e7b{HTTP/1.1, (http/1.1)}{0.0.0.0:39709} 2023-10-02T12:29:42,205 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] server.Server(415): Started @9651ms 2023-10-02T12:29:42,205 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2388): Adding backup master ZNode /1/backup-masters/jenkins-hbase3.apache.org,40619,1696249778965 2023-10-02T12:29:42,220 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:29:42,220 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:29:42,221 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase3.apache.org,40619,1696249778965 2023-10-02T12:29:42,228 DEBUG [M:0;jenkins-hbase3:40619 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase3:40619 2023-10-02T12:29:42,242 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-10-02T12:29:42,242 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-10-02T12:29:42,242 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:29:42,242 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:29:42,243 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on existing znode=/1/master 2023-10-02T12:29:42,245 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on existing znode=/1/master 2023-10-02T12:29:42,245 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase3.apache.org,40619,1696249778965 from backup master directory 2023-10-02T12:29:42,248 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase3.apache.org,40619,1696249778965 2023-10-02T12:29:42,249 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:29:42,249 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-10-02T12:29:42,249 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-10-02T12:29:42,250 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase3.apache.org,40619,1696249778965 2023-10-02T12:29:42,254 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-10-02T12:29:42,256 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-10-02T12:29:42,357 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase.id with ID: 922ab8c6-4aab-45ee-a4de-48aac3553e01 2023-10-02T12:29:42,400 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-10-02T12:29:42,419 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:29:42,419 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:29:42,463 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:29:42,466 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-10-02T12:29:42,486 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-10-02T12:29:42,486 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-10-02T12:29:42,487 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:29:42,499 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:951) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2413) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:574) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:571) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:29:42,500 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:29:43,014 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/data/master/store 2023-10-02T12:29:43,039 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-10-02T12:29:43,040 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:29:43,040 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-10-02T12:29:43,040 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:29:43,041 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:29:43,041 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-10-02T12:29:43,041 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:29:43,041 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:29:43,041 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:29:43,043 WARN [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/data/master/store/.initializing 2023-10-02T12:29:43,044 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/WALs/jenkins-hbase3.apache.org,40619,1696249778965 2023-10-02T12:29:43,053 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-10-02T12:29:43,067 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C40619%2C1696249778965, suffix=, logDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/WALs/jenkins-hbase3.apache.org,40619,1696249778965, archiveDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/oldWALs, maxLogs=10 2023-10-02T12:29:43,095 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/WALs/jenkins-hbase3.apache.org,40619,1696249778965/jenkins-hbase3.apache.org%2C40619%2C1696249778965.1696249783073, exclude list is [], retry=0 2023-10-02T12:29:43,122 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:29:43,129 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:489) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:397) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.4.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.4.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-10-02T12:29:43,201 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/WALs/jenkins-hbase3.apache.org,40619,1696249778965/jenkins-hbase3.apache.org%2C40619%2C1696249778965.1696249783073 2023-10-02T12:29:43,202 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:29:43,203 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:29:43,203 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:29:43,208 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:29:43,209 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:29:43,281 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:29:43,316 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-10-02T12:29:43,322 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:43,326 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:29:43,326 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:29:43,335 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-10-02T12:29:43,335 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:43,336 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:29:43,337 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:29:43,340 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-10-02T12:29:43,340 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:43,341 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:29:43,341 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:29:43,345 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-10-02T12:29:43,345 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:43,346 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:29:43,352 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:29:43,359 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:29:43,381 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-10-02T12:29:43,387 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-10-02T12:29:43,395 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:29:43,396 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10113887520, jitterRate=-0.058070823550224304}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-10-02T12:29:43,406 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:29:43,408 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-10-02T12:29:43,450 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61caea62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:29:43,506 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(860): No meta location available on zookeeper, skip migrating... 2023-10-02T12:29:43,528 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-10-02T12:29:43,528 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-10-02T12:29:43,531 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-10-02T12:29:43,533 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-10-02T12:29:43,539 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 6 msec 2023-10-02T12:29:43,540 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-10-02T12:29:43,577 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-10-02T12:29:43,595 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-10-02T12:29:43,600 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-10-02T12:29:43,604 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-10-02T12:29:43,606 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-10-02T12:29:43,608 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-10-02T12:29:43,611 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-10-02T12:29:43,618 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-10-02T12:29:43,620 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-10-02T12:29:43,621 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-10-02T12:29:43,622 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-10-02T12:29:43,638 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-10-02T12:29:43,641 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-10-02T12:29:43,647 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-10-02T12:29:43,647 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-10-02T12:29:43,647 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:29:43,647 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:29:43,648 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(804): Active/primary master=jenkins-hbase3.apache.org,40619,1696249778965, sessionid=0x1017326ef4b0000, setting cluster-up flag (Was=false) 2023-10-02T12:29:43,658 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-10-02T12:29:43,666 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:29:43,666 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:29:43,672 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-10-02T12:29:43,674 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,40619,1696249778965 2023-10-02T12:29:43,678 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:29:43,678 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:29:43,687 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-10-02T12:29:43,689 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,40619,1696249778965 2023-10-02T12:29:43,696 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:29:43,696 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:29:43,701 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-10-02T12:29:43,703 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase3.apache.org,40619,1696249778965 2023-10-02T12:29:43,812 DEBUG [RS:0;jenkins-hbase3:38255 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase3:38255 2023-10-02T12:29:43,822 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(997): ClusterId : 922ab8c6-4aab-45ee-a4de-48aac3553e01 2023-10-02T12:29:43,826 INFO [RS:0;jenkins-hbase3:38255 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-10-02T12:29:43,829 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-10-02T12:29:43,841 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-10-02T12:29:43,841 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-10-02T12:29:43,845 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-10-02T12:29:43,845 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-10-02T12:29:43,850 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-10-02T12:29:43,854 DEBUG [RS:0;jenkins-hbase3:38255 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5eac79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:29:43,857 DEBUG [RS:0;jenkins-hbase3:38255 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dfbe677, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-10-02T12:29:43,864 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-10-02T12:29:43,864 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-10-02T12:29:43,864 DEBUG [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-10-02T12:29:43,870 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase3.apache.org,40619,1696249778965 with isa=jenkins-hbase3.apache.org/172.31.12.81:38255, startcode=1696249782114 2023-10-02T12:29:43,874 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-10-02T12:29:43,884 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-10-02T12:29:43,889 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-10-02T12:29:43,897 DEBUG [RS:0;jenkins-hbase3:38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-10-02T12:29:43,898 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase3.apache.org,40619,1696249778965 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-10-02T12:29:43,903 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:29:43,903 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:29:43,903 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:29:43,903 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=5, maxPoolSize=5 2023-10-02T12:29:43,903 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase3:0, corePoolSize=10, maxPoolSize=10 2023-10-02T12:29:43,903 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:29:43,903 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-10-02T12:29:43,904 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:29:43,906 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1696249813906 2023-10-02T12:29:43,911 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-10-02T12:29:43,915 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-10-02T12:29:43,916 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-10-02T12:29:43,917 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-10-02T12:29:43,922 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:43,922 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-10-02T12:29:43,927 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-10-02T12:29:43,927 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-10-02T12:29:43,928 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-10-02T12:29:43,929 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-10-02T12:29:43,929 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-10-02T12:29:43,930 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-10-02T12:29:43,932 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-10-02T12:29:43,938 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-10-02T12:29:43,942 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-10-02T12:29:43,953 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6eb5489a to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:29:43,989 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c15fc8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:29:43,990 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-10-02T12:29:43,993 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-10-02T12:29:43,993 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-10-02T12:29:43,997 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696249783996,5,FailOnTimeoutGroup] 2023-10-02T12:29:43,998 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696249783997,5,FailOnTimeoutGroup] 2023-10-02T12:29:43,998 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-10-02T12:29:43,998 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1635): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-10-02T12:29:44,000 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-10-02T12:29:44,000 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-10-02T12:29:44,013 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:52131, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-10-02T12:29:44,028 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40619 {}] master.ServerManager(403): Registering regionserver=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:44,057 DEBUG [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:29:44,057 DEBUG [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:37823 2023-10-02T12:29:44,057 DEBUG [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=42929 2023-10-02T12:29:44,065 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-10-02T12:29:44,066 DEBUG [RS:0;jenkins-hbase3:38255 {}] zookeeper.ZKUtil(111): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:44,066 WARN [RS:0;jenkins-hbase3:38255 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-10-02T12:29:44,066 INFO [RS:0;jenkins-hbase3:38255 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:29:44,067 DEBUG [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:44,071 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase3.apache.org,38255,1696249782114] 2023-10-02T12:29:44,092 DEBUG [RS:0;jenkins-hbase3:38255 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-10-02T12:29:44,113 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-10-02T12:29:44,128 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-10-02T12:29:44,133 INFO [RS:0;jenkins-hbase3:38255 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-10-02T12:29:44,133 INFO [RS:0;jenkins-hbase3:38255 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:29:44,134 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-10-02T12:29:44,144 INFO [RS:0;jenkins-hbase3:38255 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-10-02T12:29:44,145 DEBUG [RS:0;jenkins-hbase3:38255 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:29:44,145 DEBUG [RS:0;jenkins-hbase3:38255 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:29:44,145 DEBUG [RS:0;jenkins-hbase3:38255 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:29:44,145 DEBUG [RS:0;jenkins-hbase3:38255 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:29:44,145 DEBUG [RS:0;jenkins-hbase3:38255 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:29:44,145 DEBUG [RS:0;jenkins-hbase3:38255 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase3:0, corePoolSize=2, maxPoolSize=2 2023-10-02T12:29:44,146 DEBUG [RS:0;jenkins-hbase3:38255 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:29:44,146 DEBUG [RS:0;jenkins-hbase3:38255 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:29:44,146 DEBUG [RS:0;jenkins-hbase3:38255 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:29:44,146 DEBUG [RS:0;jenkins-hbase3:38255 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:29:44,146 DEBUG [RS:0;jenkins-hbase3:38255 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase3:0, corePoolSize=1, maxPoolSize=1 2023-10-02T12:29:44,146 DEBUG [RS:0;jenkins-hbase3:38255 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-10-02T12:29:44,146 DEBUG [RS:0;jenkins-hbase3:38255 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase3:0, corePoolSize=3, maxPoolSize=3 2023-10-02T12:29:44,147 INFO [RS:0;jenkins-hbase3:38255 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-10-02T12:29:44,148 INFO [RS:0;jenkins-hbase3:38255 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-10-02T12:29:44,148 INFO [RS:0;jenkins-hbase3:38255 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-10-02T12:29:44,148 INFO [RS:0;jenkins-hbase3:38255 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-10-02T12:29:44,148 INFO [RS:0;jenkins-hbase3:38255 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,38255,1696249782114-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-10-02T12:29:44,167 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-10-02T12:29:44,169 INFO [RS:0;jenkins-hbase3:38255 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,38255,1696249782114-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:29:44,186 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.Replication(204): jenkins-hbase3.apache.org,38255,1696249782114 started 2023-10-02T12:29:44,186 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase3.apache.org,38255,1696249782114, RpcServer on jenkins-hbase3.apache.org/172.31.12.81:38255, sessionid=0x1017326ef4b0001 2023-10-02T12:29:44,187 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-10-02T12:29:44,187 DEBUG [RS:0;jenkins-hbase3:38255 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:44,187 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,38255,1696249782114' 2023-10-02T12:29:44,187 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-10-02T12:29:44,188 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-10-02T12:29:44,189 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-10-02T12:29:44,189 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-10-02T12:29:44,189 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,38255,1696249782114' 2023-10-02T12:29:44,189 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:29:44,189 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:29:44,190 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-10-02T12:29:44,190 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-10-02T12:29:44,190 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-10-02T12:29:44,190 DEBUG [RS:0;jenkins-hbase3:38255 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:44,190 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase3.apache.org,38255,1696249782114' 2023-10-02T12:29:44,190 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-10-02T12:29:44,191 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-10-02T12:29:44,191 DEBUG [RS:0;jenkins-hbase3:38255 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-10-02T12:29:44,191 INFO [RS:0;jenkins-hbase3:38255 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-10-02T12:29:44,192 INFO [RS:0;jenkins-hbase3:38255 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-10-02T12:29:44,302 INFO [RS:0;jenkins-hbase3:38255 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-10-02T12:29:44,305 INFO [RS:0;jenkins-hbase3:38255 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C38255%2C1696249782114, suffix=, logDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114, archiveDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/oldWALs, maxLogs=32 2023-10-02T12:29:44,317 DEBUG [RS:0;jenkins-hbase3:38255 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249784306, exclude list is [], retry=0 2023-10-02T12:29:44,323 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:29:44,328 INFO [RS:0;jenkins-hbase3:38255 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249784306 2023-10-02T12:29:44,328 DEBUG [RS:0;jenkins-hbase3:38255 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:29:44,386 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-10-02T12:29:44,386 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:29:44,403 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:29:44,405 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-10-02T12:29:44,408 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-10-02T12:29:44,408 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:44,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:29:44,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-10-02T12:29:44,411 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-10-02T12:29:44,412 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:44,412 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:29:44,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-10-02T12:29:44,415 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-10-02T12:29:44,415 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:44,416 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:29:44,417 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740 2023-10-02T12:29:44,418 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740 2023-10-02T12:29:44,421 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-10-02T12:29:44,423 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-10-02T12:29:44,428 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:29:44,429 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11617517600, jitterRate=0.08196564018726349}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-10-02T12:29:44,432 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-10-02T12:29:44,432 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-10-02T12:29:44,433 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-10-02T12:29:44,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-10-02T12:29:44,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-10-02T12:29:44,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-10-02T12:29:44,436 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-10-02T12:29:44,436 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-10-02T12:29:44,440 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-10-02T12:29:44,440 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-10-02T12:29:44,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-10-02T12:29:44,463 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-10-02T12:29:44,470 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-10-02T12:29:44,622 DEBUG [jenkins-hbase3:40619 {}] assignment.AssignmentManager(2303): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-10-02T12:29:44,631 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:44,638 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,38255,1696249782114, state=OPENING 2023-10-02T12:29:44,647 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-10-02T12:29:44,653 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:29:44,653 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:29:44,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:29:44,654 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:29:44,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:29:44,848 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(721): New admin connection to jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:44,851 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-10-02T12:29:44,855 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54062, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-10-02T12:29:44,868 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-10-02T12:29:44,869 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-10-02T12:29:44,870 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-10-02T12:29:44,875 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta, suffix=.meta, logDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114, archiveDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/oldWALs, maxLogs=32 2023-10-02T12:29:44,893 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249784877.meta, exclude list is [], retry=0 2023-10-02T12:29:44,899 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:29:44,906 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249784877.meta 2023-10-02T12:29:44,907 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:29:44,907 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:29:44,913 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:29:44,914 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-10-02T12:29:44,936 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-10-02T12:29:44,936 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-10-02T12:29:44,944 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-10-02T12:29:44,944 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:29:44,944 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-10-02T12:29:44,944 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-10-02T12:29:44,948 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-10-02T12:29:44,949 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-10-02T12:29:44,950 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:44,950 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:29:44,951 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-10-02T12:29:44,952 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-10-02T12:29:44,952 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:44,953 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:29:44,953 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-10-02T12:29:44,954 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-10-02T12:29:44,955 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:44,956 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-10-02T12:29:44,959 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740 2023-10-02T12:29:44,962 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740 2023-10-02T12:29:44,965 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-10-02T12:29:44,968 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-10-02T12:29:44,970 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10040939040, jitterRate=-0.06486468017101288}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-10-02T12:29:44,972 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-10-02T12:29:44,988 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1696249784839 2023-10-02T12:29:45,018 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:45,021 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase3.apache.org,38255,1696249782114, state=OPEN 2023-10-02T12:29:45,028 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-10-02T12:29:45,029 INFO [RS_OPEN_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-10-02T12:29:45,032 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-10-02T12:29:45,032 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-10-02T12:29:45,032 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:29:45,032 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-10-02T12:29:45,038 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-10-02T12:29:45,038 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase3.apache.org,38255,1696249782114 in 375 msec 2023-10-02T12:29:45,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-10-02T12:29:45,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 592 msec 2023-10-02T12:29:45,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.2800 sec 2023-10-02T12:29:45,060 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1063): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1696249785059, completionTime=-1 2023-10-02T12:29:45,060 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ServerManager(830): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-10-02T12:29:45,060 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1629): Joining cluster... 2023-10-02T12:29:45,290 DEBUG [hconnection-0x712cc456-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:29:45,294 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54078, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:29:45,310 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1641): Number of RegionServers=1 2023-10-02T12:29:45,310 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1696249845310 2023-10-02T12:29:45,310 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1696249905310 2023-10-02T12:29:45,310 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] assignment.AssignmentManager(1648): Joined the cluster in 250 msec 2023-10-02T12:29:45,335 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,40619,1696249778965-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-10-02T12:29:45,335 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,40619,1696249778965-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:29:45,335 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,40619,1696249778965-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:29:45,337 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase3:40619, period=300000, unit=MILLISECONDS is enabled. 2023-10-02T12:29:45,337 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-10-02T12:29:45,345 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-10-02T12:29:45,358 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-10-02T12:29:45,359 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(2368): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-10-02T12:29:45,369 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-10-02T12:29:45,373 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:29:45,376 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:45,378 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:29:45,397 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => d30d346a1595b3fd84f804f2773895cc, NAME => 'hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:29:45,809 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:29:45,809 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing d30d346a1595b3fd84f804f2773895cc, disabling compactions & flushes 2023-10-02T12:29:45,809 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc. 2023-10-02T12:29:45,809 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc. 2023-10-02T12:29:45,809 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc. after waiting 0 ms 2023-10-02T12:29:45,809 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc. 2023-10-02T12:29:45,809 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc. 2023-10-02T12:29:45,810 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for d30d346a1595b3fd84f804f2773895cc: 2023-10-02T12:29:45,813 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:29:45,829 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1696249785815"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249785815"}]},"ts":"1696249785815"} 2023-10-02T12:29:45,873 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:29:45,876 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:29:45,881 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249785876"}]},"ts":"1696249785876"} 2023-10-02T12:29:45,885 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-10-02T12:29:45,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=d30d346a1595b3fd84f804f2773895cc, ASSIGN}] 2023-10-02T12:29:45,894 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=d30d346a1595b3fd84f804f2773895cc, ASSIGN 2023-10-02T12:29:45,895 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=d30d346a1595b3fd84f804f2773895cc, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38255,1696249782114; forceNewPlan=false, retain=false 2023-10-02T12:29:46,046 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=d30d346a1595b3fd84f804f2773895cc, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:46,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure d30d346a1595b3fd84f804f2773895cc, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:29:46,211 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc. 2023-10-02T12:29:46,212 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => d30d346a1595b3fd84f804f2773895cc, NAME => 'hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:29:46,213 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:29:46,213 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace d30d346a1595b3fd84f804f2773895cc 2023-10-02T12:29:46,214 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:29:46,214 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for d30d346a1595b3fd84f804f2773895cc 2023-10-02T12:29:46,214 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for d30d346a1595b3fd84f804f2773895cc 2023-10-02T12:29:46,216 INFO [StoreOpener-d30d346a1595b3fd84f804f2773895cc-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region d30d346a1595b3fd84f804f2773895cc 2023-10-02T12:29:46,218 INFO [StoreOpener-d30d346a1595b3fd84f804f2773895cc-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d30d346a1595b3fd84f804f2773895cc columnFamilyName info 2023-10-02T12:29:46,218 DEBUG [StoreOpener-d30d346a1595b3fd84f804f2773895cc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:46,219 INFO [StoreOpener-d30d346a1595b3fd84f804f2773895cc-1 {}] regionserver.HStore(324): Store=d30d346a1595b3fd84f804f2773895cc/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:29:46,221 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/namespace/d30d346a1595b3fd84f804f2773895cc 2023-10-02T12:29:46,221 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/namespace/d30d346a1595b3fd84f804f2773895cc 2023-10-02T12:29:46,225 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for d30d346a1595b3fd84f804f2773895cc 2023-10-02T12:29:46,229 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/namespace/d30d346a1595b3fd84f804f2773895cc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:29:46,230 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened d30d346a1595b3fd84f804f2773895cc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11776546880, jitterRate=0.09677639603614807}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:29:46,231 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for d30d346a1595b3fd84f804f2773895cc: 2023-10-02T12:29:46,233 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc., pid=6, masterSystemTime=1696249786205 2023-10-02T12:29:46,237 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc. 2023-10-02T12:29:46,237 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc. 2023-10-02T12:29:46,238 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=d30d346a1595b3fd84f804f2773895cc, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:46,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-10-02T12:29:46,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure d30d346a1595b3fd84f804f2773895cc, server=jenkins-hbase3.apache.org,38255,1696249782114 in 192 msec 2023-10-02T12:29:46,249 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-10-02T12:29:46,249 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=d30d346a1595b3fd84f804f2773895cc, ASSIGN in 355 msec 2023-10-02T12:29:46,251 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:29:46,251 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249786251"}]},"ts":"1696249786251"} 2023-10-02T12:29:46,257 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-10-02T12:29:46,261 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:29:46,263 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 901 msec 2023-10-02T12:29:46,273 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-10-02T12:29:46,275 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-10-02T12:29:46,275 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:29:46,275 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-10-02T12:29:46,329 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-10-02T12:29:46,348 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:29:46,354 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 33 msec 2023-10-02T12:29:46,365 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-10-02T12:29:46,377 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:29:46,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 16 msec 2023-10-02T12:29:46,394 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-10-02T12:29:46,397 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-10-02T12:29:46,398 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1193): Master has completed initialization 4.147sec 2023-10-02T12:29:46,401 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-10-02T12:29:46,403 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-10-02T12:29:46,405 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-10-02T12:29:46,406 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-10-02T12:29:46,406 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-10-02T12:29:46,409 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,40619,1696249778965-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-10-02T12:29:46,410 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase3.apache.org,40619,1696249778965-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-10-02T12:29:46,427 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster {}] master.HMaster(1296): Balancer post startup initialization complete, took 0 seconds 2023-10-02T12:29:46,427 DEBUG [Listener at localhost/45111 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0ea35320 to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:29:46,429 INFO [master/jenkins-hbase3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-10-02T12:29:46,432 DEBUG [Listener at localhost/45111 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acffa64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:29:46,439 DEBUG [Listener at localhost/45111 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-10-02T12:29:46,439 DEBUG [Listener at localhost/45111 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-10-02T12:29:46,457 DEBUG [hconnection-0x3eb3393a-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:29:46,468 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54080, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:29:46,478 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase3.apache.org,40619,1696249778965 2023-10-02T12:29:46,478 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-10-02T12:29:46,478 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/test.cache.data in system properties and HBase conf 2023-10-02T12:29:46,478 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop.tmp.dir in system properties and HBase conf 2023-10-02T12:29:46,479 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop.log.dir in system properties and HBase conf 2023-10-02T12:29:46,479 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir in system properties and HBase conf 2023-10-02T12:29:46,479 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-10-02T12:29:46,479 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-10-02T12:29:46,479 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:29:46,479 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-10-02T12:29:46,479 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-10-02T12:29:46,479 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:29:46,479 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-10-02T12:29:46,479 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-10-02T12:29:46,479 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-10-02T12:29:46,479 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:29:46,479 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-10-02T12:29:46,480 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/nfs.dump.dir in system properties and HBase conf 2023-10-02T12:29:46,480 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/java.io.tmpdir in system properties and HBase conf 2023-10-02T12:29:46,480 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/dfs.journalnode.edits.dir in system properties and HBase conf 2023-10-02T12:29:46,480 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-10-02T12:29:46,480 INFO [Listener at localhost/45111 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-10-02T12:29:48,471 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:29:48,844 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:29:48,850 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_41311_jobhistory____bqfkjk/webapp 2023-10-02T12:29:49,023 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-10-02T12:29:49,023 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-10-02T12:29:49,023 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:29:49,026 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:29:49,126 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:29:49,273 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:29:49,590 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-10-02T12:29:49,635 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:41311 2023-10-02T12:29:49,985 WARN [Listener at jenkins-hbase3.apache.org/34561 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:29:50,078 WARN [Listener at jenkins-hbase3.apache.org/34561 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:29:50,079 INFO [Listener at jenkins-hbase3.apache.org/34561 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:29:50,085 INFO [Listener at jenkins-hbase3.apache.org/34561 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_40517_cluster____.5avknw/webapp 2023-10-02T12:29:50,157 INFO [Listener at jenkins-hbase3.apache.org/34561 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:29:50,157 INFO [Listener at jenkins-hbase3.apache.org/34561 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-10-02T12:29:50,158 INFO [Listener at jenkins-hbase3.apache.org/34561 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:29:50,158 INFO [Listener at jenkins-hbase3.apache.org/34561 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:29:50,167 INFO [Listener at jenkins-hbase3.apache.org/34561 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:29:50,428 INFO [Listener at jenkins-hbase3.apache.org/34561 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:29:50,486 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:29:50,583 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-10-02T12:29:50,585 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-10-02T12:29:50,586 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-10-02T12:29:50,587 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-10-02T12:29:50,589 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-10-02T12:29:50,589 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-10-02T12:29:50,590 INFO [Listener at jenkins-hbase3.apache.org/34561 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:29:50,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:29:50,591 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-10-02T12:29:50,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-02T12:29:50,591 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-10-02T12:29:50,591 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-10-02T12:29:50,591 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-10-02T12:29:50,603 INFO [Listener at jenkins-hbase3.apache.org/34561 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:40517 2023-10-02T12:29:50,964 WARN [Listener at jenkins-hbase3.apache.org/42983 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:29:50,989 WARN [Listener at jenkins-hbase3.apache.org/42983 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:29:50,990 INFO [Listener at jenkins-hbase3.apache.org/42983 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:29:50,996 INFO [Listener at jenkins-hbase3.apache.org/42983 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_46161_node____.jyueif/webapp 2023-10-02T12:29:51,050 INFO [Listener at jenkins-hbase3.apache.org/42983 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-10-02T12:29:51,051 INFO [Listener at jenkins-hbase3.apache.org/42983 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:29:51,051 INFO [Listener at jenkins-hbase3.apache.org/42983 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:29:51,051 INFO [Listener at jenkins-hbase3.apache.org/42983 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:29:51,061 INFO [Listener at jenkins-hbase3.apache.org/42983 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:29:51,082 INFO [Listener at jenkins-hbase3.apache.org/42983 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:29:51,179 INFO [Listener at jenkins-hbase3.apache.org/42983 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:29:51,192 INFO [Listener at jenkins-hbase3.apache.org/42983 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:46161 2023-10-02T12:29:51,402 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-10-02T12:29:51,411 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-10-02T12:29:51,411 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] log.Slf4jLog(67): jetty-6.1.26 2023-10-02T12:29:51,421 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/java.io.tmpdir/Jetty_jenkins.hbase3_apache_org_34435_node____.59jvik/webapp 2023-10-02T12:29:51,473 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-10-02T12:29:51,473 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-10-02T12:29:51,474 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-10-02T12:29:51,474 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-10-02T12:29:51,484 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:29:51,510 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:29:51,591 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-10-02T12:29:51,598 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:34435 2023-10-02T12:29:51,654 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-10-02T12:29:51,655 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:37823/backupUT 2023-10-02T12:29:51,671 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:29:51,676 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33296, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:29:51,689 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns1'} 2023-10-02T12:29:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-10-02T12:29:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-10-02T12:29:51,729 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:29:51,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 46 msec 2023-10-02T12:29:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=9 2023-10-02T12:29:51,814 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns2'} 2023-10-02T12:29:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-10-02T12:29:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-10-02T12:29:51,830 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:29:51,839 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 22 msec 2023-10-02T12:29:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=10 2023-10-02T12:29:51,926 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns3'} 2023-10-02T12:29:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-10-02T12:29:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-10-02T12:29:51,943 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:29:51,949 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 21 msec 2023-10-02T12:29:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=11 2023-10-02T12:29:52,037 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'ns4'} 2023-10-02T12:29:52,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-10-02T12:29:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-10-02T12:29:52,052 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:29:52,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 19 msec 2023-10-02T12:29:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=12 2023-10-02T12:29:52,163 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'test-1696249791655', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:29:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1696249791655 2023-10-02T12:29:52,168 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1696249791655 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:29:52,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "test-1696249791655" procId is: 13 2023-10-02T12:29:52,168 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:52,170 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1696249791655 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:29:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:29:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:29:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:29:52,603 INFO [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => ec65563da527c2d4de1d128d9fb4f28f, NAME => 'test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1696249791655', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:29:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:29:53,041 DEBUG [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(888): Instantiated test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:29:53,041 DEBUG [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1668): Closing ec65563da527c2d4de1d128d9fb4f28f, disabling compactions & flushes 2023-10-02T12:29:53,041 INFO [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1690): Closing region test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f. 2023-10-02T12:29:53,042 DEBUG [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f. 2023-10-02T12:29:53,042 DEBUG [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f. after waiting 0 ms 2023-10-02T12:29:53,042 DEBUG [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f. 2023-10-02T12:29:53,042 INFO [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1909): Closed test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f. 2023-10-02T12:29:53,042 DEBUG [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1622): Region close journal for ec65563da527c2d4de1d128d9fb4f28f: 2023-10-02T12:29:53,044 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1696249791655 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:29:53,044 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1696249793044"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249793044"}]},"ts":"1696249793044"} 2023-10-02T12:29:53,048 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:29:53,049 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1696249791655 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:29:53,050 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696249791655","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249793050"}]},"ts":"1696249793050"} 2023-10-02T12:29:53,053 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696249791655, state=ENABLING in hbase:meta 2023-10-02T12:29:53,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1696249791655, region=ec65563da527c2d4de1d128d9fb4f28f, ASSIGN}] 2023-10-02T12:29:53,060 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1696249791655, region=ec65563da527c2d4de1d128d9fb4f28f, ASSIGN 2023-10-02T12:29:53,062 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1696249791655, region=ec65563da527c2d4de1d128d9fb4f28f, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38255,1696249782114; forceNewPlan=false, retain=false 2023-10-02T12:29:53,218 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=ec65563da527c2d4de1d128d9fb4f28f, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:53,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure ec65563da527c2d4de1d128d9fb4f28f, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:29:53,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:29:53,385 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f. 2023-10-02T12:29:53,385 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => ec65563da527c2d4de1d128d9fb4f28f, NAME => 'test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:29:53,385 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:29:53,385 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1696249791655 ec65563da527c2d4de1d128d9fb4f28f 2023-10-02T12:29:53,386 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:29:53,386 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for ec65563da527c2d4de1d128d9fb4f28f 2023-10-02T12:29:53,386 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for ec65563da527c2d4de1d128d9fb4f28f 2023-10-02T12:29:53,388 INFO [StoreOpener-ec65563da527c2d4de1d128d9fb4f28f-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region ec65563da527c2d4de1d128d9fb4f28f 2023-10-02T12:29:53,390 INFO [StoreOpener-ec65563da527c2d4de1d128d9fb4f28f-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec65563da527c2d4de1d128d9fb4f28f columnFamilyName f 2023-10-02T12:29:53,391 DEBUG [StoreOpener-ec65563da527c2d4de1d128d9fb4f28f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:53,392 INFO [StoreOpener-ec65563da527c2d4de1d128d9fb4f28f-1 {}] regionserver.HStore(324): Store=ec65563da527c2d4de1d128d9fb4f28f/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:29:53,393 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f 2023-10-02T12:29:53,394 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f 2023-10-02T12:29:53,398 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for ec65563da527c2d4de1d128d9fb4f28f 2023-10-02T12:29:53,406 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:29:53,407 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened ec65563da527c2d4de1d128d9fb4f28f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9871884960, jitterRate=-0.08060906827449799}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:29:53,407 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for ec65563da527c2d4de1d128d9fb4f28f: 2023-10-02T12:29:53,409 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f., pid=15, masterSystemTime=1696249793376 2023-10-02T12:29:53,412 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f. 2023-10-02T12:29:53,412 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f. 2023-10-02T12:29:53,413 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=ec65563da527c2d4de1d128d9fb4f28f, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:53,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-10-02T12:29:53,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure ec65563da527c2d4de1d128d9fb4f28f, server=jenkins-hbase3.apache.org,38255,1696249782114 in 193 msec 2023-10-02T12:29:53,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-10-02T12:29:53,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1696249791655, region=ec65563da527c2d4de1d128d9fb4f28f, ASSIGN in 360 msec 2023-10-02T12:29:53,422 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1696249791655 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:29:53,423 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696249791655","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249793422"}]},"ts":"1696249793422"} 2023-10-02T12:29:53,425 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696249791655, state=ENABLED in hbase:meta 2023-10-02T12:29:53,428 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1696249791655 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:29:53,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1696249791655 in 1.2650 sec 2023-10-02T12:29:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=13 2023-10-02T12:29:54,288 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: default:test-1696249791655, procId: 13 completed 2023-10-02T12:29:54,290 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x428c0cf5 to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:29:54,300 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aaf6431, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:29:54,302 DEBUG [hconnection-0x19294187-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:29:54,306 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51956, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:29:54,314 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HRegion(8210): writing data to region test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f. with WAL disabled. Data may be lost in the event of a crash. 2023-10-02T12:29:54,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns2:test-16962497916551', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:29:54,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-16962497916551 2023-10-02T12:29:54,422 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16962497916551 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:29:54,423 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:54,423 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "test-16962497916551" procId is: 16 2023-10-02T12:29:54,424 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-16962497916551 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:29:54,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:29:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:29:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:29:54,842 INFO [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 58e28812c4354ed58ad888f86b1669f9, NAME => 'ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16962497916551', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:29:54,861 DEBUG [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:29:54,861 DEBUG [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1668): Closing 58e28812c4354ed58ad888f86b1669f9, disabling compactions & flushes 2023-10-02T12:29:54,861 INFO [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9. 2023-10-02T12:29:54,861 DEBUG [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9. 2023-10-02T12:29:54,861 DEBUG [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9. after waiting 0 ms 2023-10-02T12:29:54,861 DEBUG [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9. 2023-10-02T12:29:54,861 INFO [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9. 2023-10-02T12:29:54,861 DEBUG [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1622): Region close journal for 58e28812c4354ed58ad888f86b1669f9: 2023-10-02T12:29:54,863 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-16962497916551 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:29:54,863 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696249794863"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249794863"}]},"ts":"1696249794863"} 2023-10-02T12:29:54,866 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:29:54,867 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-16962497916551 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:29:54,867 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962497916551","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249794867"}]},"ts":"1696249794867"} 2023-10-02T12:29:54,869 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962497916551, state=ENABLING in hbase:meta 2023-10-02T12:29:54,874 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16962497916551, region=58e28812c4354ed58ad888f86b1669f9, ASSIGN}] 2023-10-02T12:29:54,875 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16962497916551, region=58e28812c4354ed58ad888f86b1669f9, ASSIGN 2023-10-02T12:29:54,876 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16962497916551, region=58e28812c4354ed58ad888f86b1669f9, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38255,1696249782114; forceNewPlan=false, retain=false 2023-10-02T12:29:55,028 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=58e28812c4354ed58ad888f86b1669f9, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:29:55,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure 58e28812c4354ed58ad888f86b1669f9, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:29:55,188 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9. 2023-10-02T12:29:55,188 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => 58e28812c4354ed58ad888f86b1669f9, NAME => 'ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:29:55,188 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:29:55,189 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962497916551 58e28812c4354ed58ad888f86b1669f9 2023-10-02T12:29:55,189 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:29:55,189 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for 58e28812c4354ed58ad888f86b1669f9 2023-10-02T12:29:55,189 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for 58e28812c4354ed58ad888f86b1669f9 2023-10-02T12:29:55,192 INFO [StoreOpener-58e28812c4354ed58ad888f86b1669f9-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 58e28812c4354ed58ad888f86b1669f9 2023-10-02T12:29:55,194 INFO [StoreOpener-58e28812c4354ed58ad888f86b1669f9-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 58e28812c4354ed58ad888f86b1669f9 columnFamilyName f 2023-10-02T12:29:55,194 DEBUG [StoreOpener-58e28812c4354ed58ad888f86b1669f9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:55,195 INFO [StoreOpener-58e28812c4354ed58ad888f86b1669f9-1 {}] regionserver.HStore(324): Store=58e28812c4354ed58ad888f86b1669f9/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:29:55,196 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9 2023-10-02T12:29:55,196 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9 2023-10-02T12:29:55,201 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for 58e28812c4354ed58ad888f86b1669f9 2023-10-02T12:29:55,204 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:29:55,205 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened 58e28812c4354ed58ad888f86b1669f9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10793021760, jitterRate=0.005178481340408325}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:29:55,206 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for 58e28812c4354ed58ad888f86b1669f9: 2023-10-02T12:29:55,207 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9., pid=18, masterSystemTime=1696249795183 2023-10-02T12:29:55,209 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9. 2023-10-02T12:29:55,210 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9. 2023-10-02T12:29:55,211 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=58e28812c4354ed58ad888f86b1669f9, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:55,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-10-02T12:29:55,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure 58e28812c4354ed58ad888f86b1669f9, server=jenkins-hbase3.apache.org,38255,1696249782114 in 182 msec 2023-10-02T12:29:55,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-10-02T12:29:55,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16962497916551, region=58e28812c4354ed58ad888f86b1669f9, ASSIGN in 343 msec 2023-10-02T12:29:55,220 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-16962497916551 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:29:55,220 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962497916551","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249795220"}]},"ts":"1696249795220"} 2023-10-02T12:29:55,222 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962497916551, state=ENABLED in hbase:meta 2023-10-02T12:29:55,225 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-16962497916551 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:29:55,227 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-16962497916551 in 806 msec 2023-10-02T12:29:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=16 2023-10-02T12:29:55,532 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns2:test-16962497916551, procId: 16 completed 2023-10-02T12:29:55,536 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HRegion(8210): writing data to region ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9. with WAL disabled. Data may be lost in the event of a crash. 2023-10-02T12:29:55,622 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns3:test-16962497916552', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:29:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-16962497916552 2023-10-02T12:29:55,625 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16962497916552 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:29:55,626 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:55,626 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "test-16962497916552" procId is: 19 2023-10-02T12:29:55,627 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-16962497916552 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:29:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:29:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:29:55,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:29:56,044 INFO [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 098a211186378b9b8d3dac0c285e5bf2, NAME => 'ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16962497916552', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:29:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:29:56,462 DEBUG [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:29:56,463 DEBUG [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1668): Closing 098a211186378b9b8d3dac0c285e5bf2, disabling compactions & flushes 2023-10-02T12:29:56,463 INFO [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2. 2023-10-02T12:29:56,463 DEBUG [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2. 2023-10-02T12:29:56,463 DEBUG [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2. after waiting 0 ms 2023-10-02T12:29:56,463 DEBUG [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2. 2023-10-02T12:29:56,463 INFO [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2. 2023-10-02T12:29:56,463 DEBUG [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1622): Region close journal for 098a211186378b9b8d3dac0c285e5bf2: 2023-10-02T12:29:56,464 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-16962497916552 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:29:56,465 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696249796465"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249796465"}]},"ts":"1696249796465"} 2023-10-02T12:29:56,467 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:29:56,468 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-16962497916552 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:29:56,469 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962497916552","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249796468"}]},"ts":"1696249796468"} 2023-10-02T12:29:56,470 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962497916552, state=ENABLING in hbase:meta 2023-10-02T12:29:56,475 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16962497916552, region=098a211186378b9b8d3dac0c285e5bf2, ASSIGN}] 2023-10-02T12:29:56,477 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16962497916552, region=098a211186378b9b8d3dac0c285e5bf2, ASSIGN 2023-10-02T12:29:56,478 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16962497916552, region=098a211186378b9b8d3dac0c285e5bf2, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38255,1696249782114; forceNewPlan=false, retain=false 2023-10-02T12:29:56,586 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-16962497916551' 2023-10-02T12:29:56,587 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1696249791655' 2023-10-02T12:29:56,629 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=098a211186378b9b8d3dac0c285e5bf2, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:56,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 098a211186378b9b8d3dac0c285e5bf2, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:29:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:29:56,790 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2. 2023-10-02T12:29:56,790 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 098a211186378b9b8d3dac0c285e5bf2, NAME => 'ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:29:56,791 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:29:56,791 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962497916552 098a211186378b9b8d3dac0c285e5bf2 2023-10-02T12:29:56,791 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:29:56,791 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 098a211186378b9b8d3dac0c285e5bf2 2023-10-02T12:29:56,791 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 098a211186378b9b8d3dac0c285e5bf2 2023-10-02T12:29:56,793 INFO [StoreOpener-098a211186378b9b8d3dac0c285e5bf2-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 098a211186378b9b8d3dac0c285e5bf2 2023-10-02T12:29:56,795 INFO [StoreOpener-098a211186378b9b8d3dac0c285e5bf2-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 098a211186378b9b8d3dac0c285e5bf2 columnFamilyName f 2023-10-02T12:29:56,796 DEBUG [StoreOpener-098a211186378b9b8d3dac0c285e5bf2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:56,796 INFO [StoreOpener-098a211186378b9b8d3dac0c285e5bf2-1 {}] regionserver.HStore(324): Store=098a211186378b9b8d3dac0c285e5bf2/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:29:56,797 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/098a211186378b9b8d3dac0c285e5bf2 2023-10-02T12:29:56,798 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/098a211186378b9b8d3dac0c285e5bf2 2023-10-02T12:29:56,801 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 098a211186378b9b8d3dac0c285e5bf2 2023-10-02T12:29:56,804 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/098a211186378b9b8d3dac0c285e5bf2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:29:56,804 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 098a211186378b9b8d3dac0c285e5bf2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10184948960, jitterRate=-0.05145271122455597}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:29:56,805 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 098a211186378b9b8d3dac0c285e5bf2: 2023-10-02T12:29:56,806 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2., pid=21, masterSystemTime=1696249796785 2023-10-02T12:29:56,808 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2. 2023-10-02T12:29:56,809 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2. 2023-10-02T12:29:56,809 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=098a211186378b9b8d3dac0c285e5bf2, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:56,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-10-02T12:29:56,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 098a211186378b9b8d3dac0c285e5bf2, server=jenkins-hbase3.apache.org,38255,1696249782114 in 179 msec 2023-10-02T12:29:56,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-10-02T12:29:56,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16962497916552, region=098a211186378b9b8d3dac0c285e5bf2, ASSIGN in 339 msec 2023-10-02T12:29:56,817 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-16962497916552 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:29:56,818 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962497916552","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249796817"}]},"ts":"1696249796817"} 2023-10-02T12:29:56,819 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962497916552, state=ENABLED in hbase:meta 2023-10-02T12:29:56,823 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-16962497916552 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:29:56,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-16962497916552 in 1.2010 sec 2023-10-02T12:29:57,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=19 2023-10-02T12:29:57,737 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns3:test-16962497916552, procId: 19 completed 2023-10-02T12:29:57,737 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-16962497916552 get assigned. Timeout = 60000ms 2023-10-02T12:29:57,740 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:29:57,747 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-16962497916552 assigned to meta. Checking AM states. 2023-10-02T12:29:57,748 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:29:57,748 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-16962497916552 assigned. 2023-10-02T12:29:57,751 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns4:test-16962497916553', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:29:57,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-16962497916553 2023-10-02T12:29:57,755 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16962497916553 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:29:57,755 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:57,755 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns4" qualifier: "test-16962497916553" procId is: 22 2023-10-02T12:29:57,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:29:57,756 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-16962497916553 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:29:57,772 INFO [RegionOpenAndInit-ns4:test-16962497916553-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 5ff54f3093326c5aa5ae7ad49096dbf7, NAME => 'ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-16962497916553', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:29:57,790 DEBUG [RegionOpenAndInit-ns4:test-16962497916553-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:29:57,790 DEBUG [RegionOpenAndInit-ns4:test-16962497916553-pool-0 {}] regionserver.HRegion(1668): Closing 5ff54f3093326c5aa5ae7ad49096dbf7, disabling compactions & flushes 2023-10-02T12:29:57,790 INFO [RegionOpenAndInit-ns4:test-16962497916553-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7. 2023-10-02T12:29:57,790 DEBUG [RegionOpenAndInit-ns4:test-16962497916553-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7. 2023-10-02T12:29:57,790 DEBUG [RegionOpenAndInit-ns4:test-16962497916553-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7. after waiting 0 ms 2023-10-02T12:29:57,790 DEBUG [RegionOpenAndInit-ns4:test-16962497916553-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7. 2023-10-02T12:29:57,790 INFO [RegionOpenAndInit-ns4:test-16962497916553-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7. 2023-10-02T12:29:57,790 DEBUG [RegionOpenAndInit-ns4:test-16962497916553-pool-0 {}] regionserver.HRegion(1622): Region close journal for 5ff54f3093326c5aa5ae7ad49096dbf7: 2023-10-02T12:29:57,792 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-16962497916553 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:29:57,792 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696249797792"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249797792"}]},"ts":"1696249797792"} 2023-10-02T12:29:57,795 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:29:57,796 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-16962497916553 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:29:57,797 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16962497916553","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249797796"}]},"ts":"1696249797796"} 2023-10-02T12:29:57,799 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16962497916553, state=ENABLING in hbase:meta 2023-10-02T12:29:57,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16962497916553, region=5ff54f3093326c5aa5ae7ad49096dbf7, ASSIGN}] 2023-10-02T12:29:57,807 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-16962497916553, region=5ff54f3093326c5aa5ae7ad49096dbf7, ASSIGN 2023-10-02T12:29:57,808 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-16962497916553, region=5ff54f3093326c5aa5ae7ad49096dbf7, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38255,1696249782114; forceNewPlan=false, retain=false 2023-10-02T12:29:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:29:57,959 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=5ff54f3093326c5aa5ae7ad49096dbf7, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:57,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure 5ff54f3093326c5aa5ae7ad49096dbf7, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:29:58,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:29:58,120 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7. 2023-10-02T12:29:58,120 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => 5ff54f3093326c5aa5ae7ad49096dbf7, NAME => 'ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:29:58,121 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:29:58,121 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962497916553 5ff54f3093326c5aa5ae7ad49096dbf7 2023-10-02T12:29:58,121 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:29:58,121 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for 5ff54f3093326c5aa5ae7ad49096dbf7 2023-10-02T12:29:58,121 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for 5ff54f3093326c5aa5ae7ad49096dbf7 2023-10-02T12:29:58,123 INFO [StoreOpener-5ff54f3093326c5aa5ae7ad49096dbf7-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 5ff54f3093326c5aa5ae7ad49096dbf7 2023-10-02T12:29:58,125 INFO [StoreOpener-5ff54f3093326c5aa5ae7ad49096dbf7-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5ff54f3093326c5aa5ae7ad49096dbf7 columnFamilyName f 2023-10-02T12:29:58,125 DEBUG [StoreOpener-5ff54f3093326c5aa5ae7ad49096dbf7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:58,126 INFO [StoreOpener-5ff54f3093326c5aa5ae7ad49096dbf7-1 {}] regionserver.HStore(324): Store=5ff54f3093326c5aa5ae7ad49096dbf7/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:29:58,127 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns4/test-16962497916553/5ff54f3093326c5aa5ae7ad49096dbf7 2023-10-02T12:29:58,128 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns4/test-16962497916553/5ff54f3093326c5aa5ae7ad49096dbf7 2023-10-02T12:29:58,131 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for 5ff54f3093326c5aa5ae7ad49096dbf7 2023-10-02T12:29:58,134 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns4/test-16962497916553/5ff54f3093326c5aa5ae7ad49096dbf7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:29:58,135 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened 5ff54f3093326c5aa5ae7ad49096dbf7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10297831200, jitterRate=-0.040939733386039734}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:29:58,135 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for 5ff54f3093326c5aa5ae7ad49096dbf7: 2023-10-02T12:29:58,136 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7., pid=24, masterSystemTime=1696249798115 2023-10-02T12:29:58,139 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7. 2023-10-02T12:29:58,139 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7. 2023-10-02T12:29:58,140 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=5ff54f3093326c5aa5ae7ad49096dbf7, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:58,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-10-02T12:29:58,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure 5ff54f3093326c5aa5ae7ad49096dbf7, server=jenkins-hbase3.apache.org,38255,1696249782114 in 180 msec 2023-10-02T12:29:58,147 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-10-02T12:29:58,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-16962497916553, region=5ff54f3093326c5aa5ae7ad49096dbf7, ASSIGN in 341 msec 2023-10-02T12:29:58,148 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-16962497916553 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:29:58,149 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-16962497916553","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249798149"}]},"ts":"1696249798149"} 2023-10-02T12:29:58,151 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-16962497916553, state=ENABLED in hbase:meta 2023-10-02T12:29:58,154 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-16962497916553 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:29:58,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-16962497916553 in 403 msec 2023-10-02T12:29:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=22 2023-10-02T12:29:58,362 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: ns4:test-16962497916553, procId: 22 completed 2023-10-02T12:29:58,362 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-16962497916553 get assigned. Timeout = 60000ms 2023-10-02T12:29:58,363 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:29:58,367 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-16962497916553 assigned to meta. Checking AM states. 2023-10-02T12:29:58,367 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-10-02T12:29:58,367 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-16962497916553 assigned. 2023-10-02T12:29:58,368 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x428c0cf5 to 127.0.0.1:57768 2023-10-02T12:29:58,368 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:29:58,416 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleOverwrite Thread=583, OpenFileDescriptor=660, MaxFileDescriptor=60000, SystemLoadAverage=576, ProcessCount=180, AvailableMemoryMB=5218 2023-10-02T12:29:58,416 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=583 is superior to 500 2023-10-02T12:29:58,419 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestFullRestore(216): create full backup image on multiple tables 2023-10-02T12:29:58,421 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0ee8823a to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:29:58,427 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@195ea476, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:29:58,440 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:29:58,442 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51966, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:29:58,456 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:29:58,457 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33304, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:29:58,468 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.HMaster$17(3367): Client=jenkins//172.31.12.81 creating {NAME => 'backup'} 2023-10-02T12:29:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-10-02T12:29:58,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-10-02T12:29:58,487 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-10-02T12:29:58,492 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 22 msec 2023-10-02T12:29:58,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=25 2023-10-02T12:29:58,593 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:29:58,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-10-02T12:29:58,598 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:29:58,598 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:58,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-10-02T12:29:58,599 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:29:58,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:29:58,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:29:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:29:58,916 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:29:59,022 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 7ec576d712d0f3b0c0592584b88ce987, NAME => 'backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:29:59,036 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:29:59,036 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 7ec576d712d0f3b0c0592584b88ce987, disabling compactions & flushes 2023-10-02T12:29:59,036 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:29:59,037 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:29:59,037 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. after waiting 0 ms 2023-10-02T12:29:59,037 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:29:59,037 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:29:59,037 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 7ec576d712d0f3b0c0592584b88ce987: 2023-10-02T12:29:59,039 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:29:59,039 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1696249799039"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249799039"}]},"ts":"1696249799039"} 2023-10-02T12:29:59,041 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:29:59,043 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:29:59,043 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249799043"}]},"ts":"1696249799043"} 2023-10-02T12:29:59,045 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-10-02T12:29:59,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=7ec576d712d0f3b0c0592584b88ce987, ASSIGN}] 2023-10-02T12:29:59,054 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=7ec576d712d0f3b0c0592584b88ce987, ASSIGN 2023-10-02T12:29:59,055 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=7ec576d712d0f3b0c0592584b88ce987, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38255,1696249782114; forceNewPlan=false, retain=false 2023-10-02T12:29:59,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:29:59,207 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=7ec576d712d0f3b0c0592584b88ce987, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:59,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure 7ec576d712d0f3b0c0592584b88ce987, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:29:59,369 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:29:59,369 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => 7ec576d712d0f3b0c0592584b88ce987, NAME => 'backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:29:59,370 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:29:59,370 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:29:59,370 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:29:59,370 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:29:59,370 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:29:59,372 INFO [StoreOpener-7ec576d712d0f3b0c0592584b88ce987-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:29:59,376 INFO [StoreOpener-7ec576d712d0f3b0c0592584b88ce987-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ec576d712d0f3b0c0592584b88ce987 columnFamilyName meta 2023-10-02T12:29:59,376 DEBUG [StoreOpener-7ec576d712d0f3b0c0592584b88ce987-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:59,377 INFO [StoreOpener-7ec576d712d0f3b0c0592584b88ce987-1 {}] regionserver.HStore(324): Store=7ec576d712d0f3b0c0592584b88ce987/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:29:59,377 INFO [StoreOpener-7ec576d712d0f3b0c0592584b88ce987-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:29:59,379 INFO [StoreOpener-7ec576d712d0f3b0c0592584b88ce987-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7ec576d712d0f3b0c0592584b88ce987 columnFamilyName session 2023-10-02T12:29:59,379 DEBUG [StoreOpener-7ec576d712d0f3b0c0592584b88ce987-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:59,380 INFO [StoreOpener-7ec576d712d0f3b0c0592584b88ce987-1 {}] regionserver.HStore(324): Store=7ec576d712d0f3b0c0592584b88ce987/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:29:59,381 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:29:59,382 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:29:59,384 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:29:59,386 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:29:59,390 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:29:59,391 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened 7ec576d712d0f3b0c0592584b88ce987; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10817035680, jitterRate=0.007414951920509338}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:29:59,392 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for 7ec576d712d0f3b0c0592584b88ce987: 2023-10-02T12:29:59,393 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987., pid=28, masterSystemTime=1696249799363 2023-10-02T12:29:59,395 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:29:59,395 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:29:59,396 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=7ec576d712d0f3b0c0592584b88ce987, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:59,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-10-02T12:29:59,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure 7ec576d712d0f3b0c0592584b88ce987, server=jenkins-hbase3.apache.org,38255,1696249782114 in 188 msec 2023-10-02T12:29:59,404 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-10-02T12:29:59,404 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=7ec576d712d0f3b0c0592584b88ce987, ASSIGN in 349 msec 2023-10-02T12:29:59,405 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:29:59,405 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249799405"}]},"ts":"1696249799405"} 2023-10-02T12:29:59,407 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-10-02T12:29:59,409 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:29:59,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 816 msec 2023-10-02T12:29:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=26 2023-10-02T12:29:59,707 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-10-02T12:29:59,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:29:59,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-10-02T12:29:59,726 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:29:59,726 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:29:59,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-10-02T12:29:59,728 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:29:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:29:59,742 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 286cb7fdcd7cb6dd93ac8373e6047f98, NAME => 'backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:29:59,759 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:29:59,759 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing 286cb7fdcd7cb6dd93ac8373e6047f98, disabling compactions & flushes 2023-10-02T12:29:59,759 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98. 2023-10-02T12:29:59,759 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98. 2023-10-02T12:29:59,760 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98. after waiting 0 ms 2023-10-02T12:29:59,760 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98. 2023-10-02T12:29:59,760 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98. 2023-10-02T12:29:59,760 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for 286cb7fdcd7cb6dd93ac8373e6047f98: 2023-10-02T12:29:59,761 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:29:59,762 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1696249799761"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249799761"}]},"ts":"1696249799761"} 2023-10-02T12:29:59,764 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:29:59,765 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:29:59,765 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249799765"}]},"ts":"1696249799765"} 2023-10-02T12:29:59,767 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-10-02T12:29:59,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=286cb7fdcd7cb6dd93ac8373e6047f98, ASSIGN}] 2023-10-02T12:29:59,773 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=286cb7fdcd7cb6dd93ac8373e6047f98, ASSIGN 2023-10-02T12:29:59,774 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=286cb7fdcd7cb6dd93ac8373e6047f98, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38255,1696249782114; forceNewPlan=false, retain=false 2023-10-02T12:29:59,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:29:59,926 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=286cb7fdcd7cb6dd93ac8373e6047f98, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:29:59,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure 286cb7fdcd7cb6dd93ac8373e6047f98, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:30:00,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:30:00,087 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98. 2023-10-02T12:30:00,087 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => 286cb7fdcd7cb6dd93ac8373e6047f98, NAME => 'backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:30:00,088 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:30:00,088 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk 286cb7fdcd7cb6dd93ac8373e6047f98 2023-10-02T12:30:00,088 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:30:00,088 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for 286cb7fdcd7cb6dd93ac8373e6047f98 2023-10-02T12:30:00,088 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for 286cb7fdcd7cb6dd93ac8373e6047f98 2023-10-02T12:30:00,090 INFO [StoreOpener-286cb7fdcd7cb6dd93ac8373e6047f98-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 286cb7fdcd7cb6dd93ac8373e6047f98 2023-10-02T12:30:00,092 INFO [StoreOpener-286cb7fdcd7cb6dd93ac8373e6047f98-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 286cb7fdcd7cb6dd93ac8373e6047f98 columnFamilyName meta 2023-10-02T12:30:00,092 DEBUG [StoreOpener-286cb7fdcd7cb6dd93ac8373e6047f98-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:30:00,093 INFO [StoreOpener-286cb7fdcd7cb6dd93ac8373e6047f98-1 {}] regionserver.HStore(324): Store=286cb7fdcd7cb6dd93ac8373e6047f98/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:30:00,093 INFO [StoreOpener-286cb7fdcd7cb6dd93ac8373e6047f98-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 286cb7fdcd7cb6dd93ac8373e6047f98 2023-10-02T12:30:00,095 INFO [StoreOpener-286cb7fdcd7cb6dd93ac8373e6047f98-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 286cb7fdcd7cb6dd93ac8373e6047f98 columnFamilyName session 2023-10-02T12:30:00,095 DEBUG [StoreOpener-286cb7fdcd7cb6dd93ac8373e6047f98-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:30:00,096 INFO [StoreOpener-286cb7fdcd7cb6dd93ac8373e6047f98-1 {}] regionserver.HStore(324): Store=286cb7fdcd7cb6dd93ac8373e6047f98/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:30:00,097 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system_bulk/286cb7fdcd7cb6dd93ac8373e6047f98 2023-10-02T12:30:00,097 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system_bulk/286cb7fdcd7cb6dd93ac8373e6047f98 2023-10-02T12:30:00,099 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-10-02T12:30:00,101 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for 286cb7fdcd7cb6dd93ac8373e6047f98 2023-10-02T12:30:00,104 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system_bulk/286cb7fdcd7cb6dd93ac8373e6047f98/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:30:00,105 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened 286cb7fdcd7cb6dd93ac8373e6047f98; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10656985280, jitterRate=-0.00749090313911438}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-10-02T12:30:00,105 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for 286cb7fdcd7cb6dd93ac8373e6047f98: 2023-10-02T12:30:00,107 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98., pid=31, masterSystemTime=1696249800082 2023-10-02T12:30:00,109 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98. 2023-10-02T12:30:00,109 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98. 2023-10-02T12:30:00,110 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=286cb7fdcd7cb6dd93ac8373e6047f98, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:00,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-10-02T12:30:00,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure 286cb7fdcd7cb6dd93ac8373e6047f98, server=jenkins-hbase3.apache.org,38255,1696249782114 in 186 msec 2023-10-02T12:30:00,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=30, resume processing ppid=29 2023-10-02T12:30:00,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=286cb7fdcd7cb6dd93ac8373e6047f98, ASSIGN in 347 msec 2023-10-02T12:30:00,122 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:30:00,122 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249800122"}]},"ts":"1696249800122"} 2023-10-02T12:30:00,124 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-10-02T12:30:00,128 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:30:00,130 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 406 msec 2023-10-02T12:30:00,134 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-10-02T12:30:00,134 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-10-02T12:30:00,135 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-10-02T12:30:00,135 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-10-02T12:30:00,136 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1696249791655 2023-10-02T12:30:00,137 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1696249791655 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:30:00,138 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16962497916551 2023-10-02T12:30:00,138 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-16962497916551 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:30:00,139 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16962497916552 2023-10-02T12:30:00,139 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-16962497916552 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:30:00,140 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16962497916553 2023-10-02T12:30:00,140 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-16962497916553 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:30:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=29 2023-10-02T12:30:00,335 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$TableFuture(3677): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-10-02T12:30:00,353 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:30:00,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:30:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249800416 (current time:1696249800416). 2023-10-02T12:30:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:30:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:30:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3768b1bf to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:30:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@151c2293, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:00,439 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43508, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3768b1bf to 127.0.0.1:57768 2023-10-02T12:30:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:30:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:30:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-10-02T12:30:00,458 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:30:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:30:00,464 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:30:00,480 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:30:00,483 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:30:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:30:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:30:00,897 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:30:00,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987}] 2023-10-02T12:30:00,912 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:30:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:30:01,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-10-02T12:30:01,094 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:30:01,101 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing 7ec576d712d0f3b0c0592584b88ce987 2/2 column families, dataSize=45 B heapSize=632 B 2023-10-02T12:30:01,269 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/542bcb858b3b482dbbc3e2620ea1c1c0 is 49, key is activesession:/session:c/1696249800396/Put/seqid=0 2023-10-02T12:30:01,304 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/542bcb858b3b482dbbc3e2620ea1c1c0 2023-10-02T12:30:01,452 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/542bcb858b3b482dbbc3e2620ea1c1c0 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0 2023-10-02T12:30:01,471 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0, entries=1, sequenceid=5, filesize=4.9 K 2023-10-02T12:30:01,481 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 7ec576d712d0f3b0c0592584b88ce987 in 381ms, sequenceid=5, compaction requested=false 2023-10-02T12:30:01,481 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-10-02T12:30:01,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for 7ec576d712d0f3b0c0592584b88ce987: 2023-10-02T12:30:01,484 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. for snapshot_backup_system completed. 2023-10-02T12:30:01,485 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:30:01,516 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:30:01,524 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:30:01,524 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0] hfiles 2023-10-02T12:30:01,524 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0 for snapshot=snapshot_backup_system 2023-10-02T12:30:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:30:01,638 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:30:01,643 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-10-02T12:30:01,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=33 2023-10-02T12:30:01,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:30:01,650 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:30:01,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-10-02T12:30:01,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 in 751 msec 2023-10-02T12:30:01,655 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:30:01,658 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:30:01,662 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:30:01,663 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:30:01,666 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:30:02,199 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:30:02,243 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:30:02,244 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:30:02,248 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:30:02,248 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-10-02T12:30:02,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.7980 sec 2023-10-02T12:30:02,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=32 2023-10-02T12:30:02,582 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-10-02T12:30:02,583 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(115): Backup backup_1696249798432 started at 1696249802582. 2023-10-02T12:30:02,588 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-10-02T12:30:02,589 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-16962497916553' 2023-10-02T12:30:02,592 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16962497916552' 2023-10-02T12:30:02,616 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(119): Backup session backup_1696249798432 has been started. 2023-10-02T12:30:02,620 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:30:02,638 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:30:02,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:30:02,647 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:30:02,647 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:30:02,648 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:30:02,648 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:30:02,649 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:02,649 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:02,651 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:02,651 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:02,652 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:30:02,654 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:30:02,654 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:30:02,654 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:30:02,656 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:02,656 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:02,657 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:30:02,658 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:02,658 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:30:02,661 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:30:02,661 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:30:02,666 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:30:02,666 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:30:02,667 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:30:02,667 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:30:02,667 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:30:02,671 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:02,671 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:02,671 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:02,671 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:30:02,671 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:30:02,672 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-10-02T12:30:02,672 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:30:02,672 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:30:02,673 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:30:02,673 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:02,674 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:02,674 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:30:02,675 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:30:02,675 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@631a9e60[Count = 0] remaining members to acquire global barrier 2023-10-02T12:30:02,675 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:30:02,675 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:02,678 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:02,678 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:02,678 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:02,678 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:30:02,678 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:02,678 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:30:02,680 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:30:02,680 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:02,681 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696249784306 highest: 1696249784306 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:02,681 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114:(num 1696249784306) roll requested 2023-10-02T12:30:02,698 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249802681, exclude list is [], retry=0 2023-10-02T12:30:02,711 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:30:02,718 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249784306 with entries=19, filesize=4.02 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249802681 2023-10-02T12:30:02,720 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:30:02,720 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249784306 is not closed yet, will try archiving it next time 2023-10-02T12:30:02,720 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta:.meta(num 1696249784877) roll requested 2023-10-02T12:30:02,739 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249802720.meta, exclude list is [], retry=0 2023-10-02T12:30:02,744 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:30:02,749 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249784877.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249802720.meta 2023-10-02T12:30:02,750 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:30:02,750 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249784877.meta is not closed yet, will try archiving it next time 2023-10-02T12:30:02,761 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249784877.meta not finished, retry = 0 2023-10-02T12:30:02,764 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 83 2023-10-02T12:30:02,764 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696249802681 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:02,765 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:30:02,767 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:53856, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-10-02T12:30:02,800 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:30:02,800 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,38255,1696249782114' in zk 2023-10-02T12:30:02,804 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:30:02,804 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:30:02,804 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:30:02,806 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:02,806 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:02,806 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:30:02,806 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:30:02,807 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:30:02,807 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:30:02,809 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:02,810 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:02,811 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:30:02,812 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:02,812 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:02,814 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,38255,1696249782114': 2023-10-02T12:30:02,814 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:30:02,814 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:30:02,814 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:30:02,814 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:30:02,814 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:30:02,814 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:30:02,821 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:30:02,821 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:30:02,821 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:30:02,821 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:30:02,821 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:02,821 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:02,821 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:30:02,821 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:30:02,822 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:30:02,822 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:30:02,823 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:02,823 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:30:02,825 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:02,825 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:02,827 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:30:02,828 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:02,829 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:02,829 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:02,830 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:30:02,831 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:02,831 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:02,847 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:30:02,847 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:30:02,847 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:02,847 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:02,847 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:30:02,847 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:02,847 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:30:02,847 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:02,847 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:02,847 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:02,847 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:30:02,847 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:30:02,847 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-10-02T12:30:02,847 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:30:02,847 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:30:02,849 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:30:02,849 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:30:02,852 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-10-02T12:30:02,852 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-10-02T12:30:02,952 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-10-02T12:30:02,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:30:02,971 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249802968_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } 2023-10-02T12:30:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249802972 (current time:1696249802972). 2023-10-02T12:30:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:30:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249802968_ns2_test-16962497916551 VERSION not specified, setting to 2 2023-10-02T12:30:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5edaabda to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:30:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7816498b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:03,009 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43510, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5edaabda to 127.0.0.1:57768 2023-10-02T12:30:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:30:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249802968_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } 2023-10-02T12:30:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249802968_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-10-02T12:30:03,026 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249802968_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:30:03,028 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249802968_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:30:03,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:30:03,032 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249802968_ns2_test-16962497916551 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:30:03,036 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249802968_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:30:03,063 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249802968_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:30:03,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 58e28812c4354ed58ad888f86b1669f9}] 2023-10-02T12:30:03,069 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 58e28812c4354ed58ad888f86b1669f9 2023-10-02T12:30:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:30:03,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-10-02T12:30:03,226 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9. 2023-10-02T12:30:03,228 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing 58e28812c4354ed58ad888f86b1669f9 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-10-02T12:30:03,255 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9/.tmp/f/a8381b3810df47eb97848d2aa7f60a19 is 37, key is row10/f:q1/1696249795545/Put/seqid=0 2023-10-02T12:30:03,286 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9/.tmp/f/a8381b3810df47eb97848d2aa7f60a19 2023-10-02T12:30:03,307 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9/.tmp/f/a8381b3810df47eb97848d2aa7f60a19 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9/f/a8381b3810df47eb97848d2aa7f60a19 2023-10-02T12:30:03,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:30:03,348 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9/f/a8381b3810df47eb97848d2aa7f60a19, entries=99, sequenceid=103, filesize=8.2 K 2023-10-02T12:30:03,359 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 58e28812c4354ed58ad888f86b1669f9 in 132ms, sequenceid=103, compaction requested=false 2023-10-02T12:30:03,360 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for 58e28812c4354ed58ad888f86b1669f9: 2023-10-02T12:30:03,360 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9. for snapshot_1696249802968_ns2_test-16962497916551 completed. 2023-10-02T12:30:03,361 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9.' region-info for snapshot=snapshot_1696249802968_ns2_test-16962497916551 2023-10-02T12:30:03,361 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:30:03,361 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9/f/a8381b3810df47eb97848d2aa7f60a19] hfiles 2023-10-02T12:30:03,361 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9/f/a8381b3810df47eb97848d2aa7f60a19 for snapshot=snapshot_1696249802968_ns2_test-16962497916551 2023-10-02T12:30:03,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:30:03,819 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9. 2023-10-02T12:30:03,819 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-10-02T12:30:03,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=35 2023-10-02T12:30:03,820 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249802968_ns2_test-16962497916551 on region 58e28812c4354ed58ad888f86b1669f9 2023-10-02T12:30:03,822 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure 58e28812c4354ed58ad888f86b1669f9 2023-10-02T12:30:03,828 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-10-02T12:30:03,828 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249802968_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:30:03,828 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure 58e28812c4354ed58ad888f86b1669f9 in 761 msec 2023-10-02T12:30:03,829 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249802968_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:30:03,830 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249802968_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:30:03,830 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249802968_ns2_test-16962497916551 2023-10-02T12:30:03,832 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249802968_ns2_test-16962497916551 2023-10-02T12:30:03,879 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249802968_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:30:03,897 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249802968_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:30:03,898 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249802968_ns2_test-16962497916551 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249802968_ns2_test-16962497916551 2023-10-02T12:30:03,903 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249802968_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:30:03,903 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249802968_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-10-02T12:30:03,906 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1696249802968_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } in 885 msec 2023-10-02T12:30:04,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=34 2023-10-02T12:30:04,139 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: ns2:test-16962497916551, procId: 34 completed 2023-10-02T12:30:04,141 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249804139_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } 2023-10-02T12:30:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249804141 (current time:1696249804141). 2023-10-02T12:30:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:30:04,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249804139_ns3_test-16962497916552 VERSION not specified, setting to 2 2023-10-02T12:30:04,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x30be0950 to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:30:04,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bb1e0e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:04,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:04,154 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43526, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:04,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x30be0950 to 127.0.0.1:57768 2023-10-02T12:30:04,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:04,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:30:04,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696249804139_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } 2023-10-02T12:30:04,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249804139_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-10-02T12:30:04,162 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696249804139_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:30:04,163 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696249804139_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:30:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:30:04,171 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249804139_ns3_test-16962497916552 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:30:04,176 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696249804139_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:30:04,208 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696249804139_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:30:04,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 098a211186378b9b8d3dac0c285e5bf2}] 2023-10-02T12:30:04,210 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 098a211186378b9b8d3dac0c285e5bf2 2023-10-02T12:30:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:30:04,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2023-10-02T12:30:04,363 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2. 2023-10-02T12:30:04,365 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2520): Flush status journal for 098a211186378b9b8d3dac0c285e5bf2: 2023-10-02T12:30:04,365 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2. for snapshot_1696249804139_ns3_test-16962497916552 completed. 2023-10-02T12:30:04,365 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2.' region-info for snapshot=snapshot_1696249804139_ns3_test-16962497916552 2023-10-02T12:30:04,365 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:30:04,365 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:30:04,397 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2. 2023-10-02T12:30:04,398 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=37 2023-10-02T12:30:04,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=37 2023-10-02T12:30:04,399 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249804139_ns3_test-16962497916552 on region 098a211186378b9b8d3dac0c285e5bf2 2023-10-02T12:30:04,402 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 098a211186378b9b8d3dac0c285e5bf2 2023-10-02T12:30:04,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=37, resume processing ppid=36 2023-10-02T12:30:04,414 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696249804139_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:30:04,414 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, ppid=36, state=SUCCESS; SnapshotRegionProcedure 098a211186378b9b8d3dac0c285e5bf2 in 198 msec 2023-10-02T12:30:04,416 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696249804139_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:30:04,417 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696249804139_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:30:04,417 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249804139_ns3_test-16962497916552 2023-10-02T12:30:04,419 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249804139_ns3_test-16962497916552 2023-10-02T12:30:04,452 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696249804139_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:30:04,463 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696249804139_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:30:04,464 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249804139_ns3_test-16962497916552 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249804139_ns3_test-16962497916552 2023-10-02T12:30:04,467 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696249804139_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:30:04,467 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249804139_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-10-02T12:30:04,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1696249804139_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } in 310 msec 2023-10-02T12:30:04,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=36 2023-10-02T12:30:04,480 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: ns3:test-16962497916552, procId: 36 completed 2023-10-02T12:30:04,480 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696249798432 2023-10-02T12:30:04,480 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:30:04,485 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-10-02T12:30:04,488 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249798432_ns2:test-16962497916551 2023-10-02T12:30:04,488 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249802968_ns2_test-16962497916551 to hdfs://localhost:37823/backupUT/backup_1696249798432/ns2/test-16962497916551/ 2023-10-02T12:30:04,503 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:30:04,570 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:30:04,571 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,40619,1696249778965_-1726041891_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37823/backupUT/backup_1696249798432/ns2/test-16962497916551, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37823/backupUT/backup_1696249798432/ns2/test-16962497916551/.hbase-snapshot/.tmp/snapshot_1696249802968_ns2_test-16962497916551 2023-10-02T12:30:04,571 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07, snapshotDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249802968_ns2_test-16962497916551. 2023-10-02T12:30:04,612 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249802968_ns2_test-16962497916551 to hdfs://localhost:37823/backupUT/backup_1696249798432/ns2/test-16962497916551/.hbase-snapshot/.tmp/snapshot_1696249802968_ns2_test-16962497916551 2023-10-02T12:30:04,880 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:30:05,109 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:05,110 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:05,111 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:05,112 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:06,717 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-2024088229015223134.jar 2023-10-02T12:30:06,718 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:06,718 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:06,719 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:06,720 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:06,720 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:06,721 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:06,726 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:06,726 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:06,727 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:06,728 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:30:06,729 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:30:06,730 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:30:06,731 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:30:06,732 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:30:06,733 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:30:06,733 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:30:06,736 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:30:06,737 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:30:06,738 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:30:06,739 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:30:06,739 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:30:06,745 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:06,746 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:06,747 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:06,747 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:06,748 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:06,749 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:06,750 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:06,851 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:30:06,999 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249802968_ns2_test-16962497916551' hfile list 2023-10-02T12:30:07,045 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-10-02T12:30:07,756 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249807543/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:30:07,756 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249807543/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:30:08,182 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:30:08,255 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9/f/a8381b3810df47eb97848d2aa7f60a19 output=hdfs://localhost:37823/backupUT/backup_1696249798432/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9/f/a8381b3810df47eb97848d2aa7f60a19 2023-10-02T12:30:08,255 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 0.890M/sec 2023-10-02T12:30:08,947 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:30:08,949 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:30:08,982 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249802968_ns2_test-16962497916551 2023-10-02T12:30:08,982 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249802968_ns2_test-16962497916551 finished. 2023-10-02T12:30:08,982 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249798432_ns3:test-16962497916552 2023-10-02T12:30:08,983 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249804139_ns3_test-16962497916552 to hdfs://localhost:37823/backupUT/backup_1696249798432/ns3/test-16962497916552/ 2023-10-02T12:30:08,983 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:30:09,023 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:30:09,023 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,40619,1696249778965_1031650289_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37823/backupUT/backup_1696249798432/ns3/test-16962497916552, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37823/backupUT/backup_1696249798432/ns3/test-16962497916552/.hbase-snapshot/.tmp/snapshot_1696249804139_ns3_test-16962497916552 2023-10-02T12:30:09,023 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07, snapshotDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249804139_ns3_test-16962497916552. 2023-10-02T12:30:09,056 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249804139_ns3_test-16962497916552 to hdfs://localhost:37823/backupUT/backup_1696249798432/ns3/test-16962497916552/.hbase-snapshot/.tmp/snapshot_1696249804139_ns3_test-16962497916552 2023-10-02T12:30:09,628 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:09,628 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:09,629 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:09,630 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:11,214 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-6564360245378440457.jar 2023-10-02T12:30:11,214 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:11,215 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:11,216 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:11,218 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:11,218 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:11,219 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:11,220 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:11,220 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:11,221 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:11,222 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:30:11,223 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:30:11,224 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:30:11,224 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:30:11,225 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:30:11,226 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:30:11,227 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:30:11,234 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:30:11,235 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:30:11,236 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:30:11,236 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:30:11,237 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:30:11,238 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:11,239 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:11,240 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:11,240 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:11,241 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:11,242 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:11,243 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:11,303 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:30:11,340 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249804139_ns3_test-16962497916552' hfile list 2023-10-02T12:30:11,479 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249811416/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:30:11,479 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249811416/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:30:12,551 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:30:12,555 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:30:12,577 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249804139_ns3_test-16962497916552 2023-10-02T12:30:12,577 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249804139_ns3_test-16962497916552 finished. 2023-10-02T12:30:12,578 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1034): ns2:test-16962497916551 2023-10-02T12:30:12,578 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1034): ns3:test-16962497916552 2023-10-02T12:30:12,703 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249798432 2023-10-02T12:30:12,704 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:30:12,726 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37823/backupUT/backup_1696249798432/.backup.manifest 2023-10-02T12:30:12,726 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249798432 2023-10-02T12:30:12,726 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:30:12,749 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37823/backupUT/backup_1696249798432/.backup.manifest 2023-10-02T12:30:12,749 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(358): Backup backup_1696249798432 finished: type=FULL,tablelist=ns2:test-16962497916551;ns3:test-16962497916552,targetRootDir=hdfs://localhost:37823/backupUT,startts=1696249802582,completets=1696249812700,bytescopied=0 2023-10-02T12:30:12,749 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:30:12,749 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249802968_ns2_test-16962497916551 2023-10-02T12:30:12,764 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249802968_ns2_test-16962497916551" 2023-10-02T12:30:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249802968_ns2_test-16962497916551 2023-10-02T12:30:12,777 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249802968_ns2_test-16962497916551 for backup backup_1696249798432 succeeded. 2023-10-02T12:30:12,777 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249804139_ns3_test-16962497916552 2023-10-02T12:30:12,779 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249804139_ns3_test-16962497916552" 2023-10-02T12:30:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249804139_ns3_test-16962497916552 2023-10-02T12:30:12,789 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249804139_ns3_test-16962497916552 for backup backup_1696249798432 succeeded. 2023-10-02T12:30:12,791 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:30:12,812 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:30:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:30:12,817 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:30:12,829 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:30:12,878 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(376): Backup backup_1696249798432 completed. 2023-10-02T12:30:12,878 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:30:12,878 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0ee8823a to 127.0.0.1:57768 2023-10-02T12:30:12,879 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:12,953 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249798432/.backup.manifest 2023-10-02T12:30:12,957 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249798432/.backup.manifest 2023-10-02T12:30:12,960 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249798432/.backup.manifest 2023-10-02T12:30:12,974 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249798432/.backup.manifest 2023-10-02T12:30:12,985 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249798432/.backup.manifest 2023-10-02T12:30:12,991 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249798432/.backup.manifest 2023-10-02T12:30:12,991 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-16962497916551' to 'ns2:test-16962497916551' from full backup image hdfs://localhost:37823/backupUT/backup_1696249798432/ns2/test-16962497916551 2023-10-02T12:30:13,019 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(476): Truncating exising target table 'ns2:test-16962497916551', preserving region splits 2023-10-02T12:30:13,021 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$18(967): Started disable of ns2:test-16962497916551 2023-10-02T12:30:13,029 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable ns2:test-16962497916551 2023-10-02T12:30:13,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=38, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:test-16962497916551 2023-10-02T12:30:13,046 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962497916551","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249813046"}]},"ts":"1696249813046"} 2023-10-02T12:30:13,052 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962497916551, state=DISABLING in hbase:meta 2023-10-02T12:30:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:30:13,066 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set ns2:test-16962497916551 to state=DISABLING 2023-10-02T12:30:13,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-16962497916551, region=58e28812c4354ed58ad888f86b1669f9, UNASSIGN}] 2023-10-02T12:30:13,076 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-16962497916551, region=58e28812c4354ed58ad888f86b1669f9, UNASSIGN 2023-10-02T12:30:13,085 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=58e28812c4354ed58ad888f86b1669f9, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:13,088 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE; CloseRegionProcedure 58e28812c4354ed58ad888f86b1669f9, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:30:13,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:30:13,257 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(119): Close 58e28812c4354ed58ad888f86b1669f9 2023-10-02T12:30:13,257 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1668): Closing 58e28812c4354ed58ad888f86b1669f9, disabling compactions & flushes 2023-10-02T12:30:13,257 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1690): Closing region ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9. 2023-10-02T12:30:13,258 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9. 2023-10-02T12:30:13,258 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9. after waiting 0 ms 2023-10-02T12:30:13,258 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9. 2023-10-02T12:30:13,279 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-10-02T12:30:13,280 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:30:13,281 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1909): Closed ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9. 2023-10-02T12:30:13,281 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1622): Region close journal for 58e28812c4354ed58ad888f86b1669f9: 2023-10-02T12:30:13,285 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=58e28812c4354ed58ad888f86b1669f9, regionState=CLOSED 2023-10-02T12:30:13,286 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(163): Closed 58e28812c4354ed58ad888f86b1669f9 2023-10-02T12:30:13,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=40, resume processing ppid=39 2023-10-02T12:30:13,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, ppid=39, state=SUCCESS; CloseRegionProcedure 58e28812c4354ed58ad888f86b1669f9, server=jenkins-hbase3.apache.org,38255,1696249782114 in 200 msec 2023-10-02T12:30:13,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=39, resume processing ppid=38 2023-10-02T12:30:13,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, ppid=38, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16962497916551, region=58e28812c4354ed58ad888f86b1669f9, UNASSIGN in 216 msec 2023-10-02T12:30:13,295 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962497916551","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249813295"}]},"ts":"1696249813295"} 2023-10-02T12:30:13,297 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962497916551, state=DISABLED in hbase:meta 2023-10-02T12:30:13,299 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set ns2:test-16962497916551 to state=DISABLED 2023-10-02T12:30:13,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, state=SUCCESS; DisableTableProcedure table=ns2:test-16962497916551 in 268 msec 2023-10-02T12:30:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=38 2023-10-02T12:30:13,362 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: ns2:test-16962497916551, procId: 38 completed 2023-10-02T12:30:13,364 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$16(847): Started truncating ns2:test-16962497916551 2023-10-02T12:30:13,372 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.HMaster$6(2480): Client=jenkins//172.31.12.81 truncate ns2:test-16962497916551 2023-10-02T12:30:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=41, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns2:test-16962497916551 preserveSplits=true) 2023-10-02T12:30:13,388 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(87): waiting for 'ns2:test-16962497916551' regions in transition 2023-10-02T12:30:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-10-02T12:30:13,402 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9 2023-10-02T12:30:13,407 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9/f, FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9/recovered.edits] 2023-10-02T12:30:13,417 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9/f/a8381b3810df47eb97848d2aa7f60a19 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/archive/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9/f/a8381b3810df47eb97848d2aa7f60a19 2023-10-02T12:30:13,422 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9/recovered.edits/106.seqid to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/archive/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9/recovered.edits/106.seqid 2023-10-02T12:30:13,423 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9 2023-10-02T12:30:13,423 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived ns2:test-16962497916551 regions 2023-10-02T12:30:13,442 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:test-16962497916551 from hbase:meta 2023-10-02T12:30:13,445 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:test-16962497916551' descriptor. 2023-10-02T12:30:13,447 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:test-16962497916551' from region states. 2023-10-02T12:30:13,447 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696249813447"}]},"ts":"9223372036854775807"} 2023-10-02T12:30:13,451 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:30:13,451 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 58e28812c4354ed58ad888f86b1669f9, NAME => 'ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:30:13,451 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:test-16962497916551' as deleted. 2023-10-02T12:30:13,451 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-16962497916551","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1696249813451"}]},"ts":"9223372036854775807"} 2023-10-02T12:30:13,454 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:test-16962497916551 state from META 2023-10-02T12:30:13,468 INFO [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => ce860946aa8cd08494bb77afea49e9de, NAME => 'ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16962497916551', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:30:13,479 DEBUG [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:30:13,479 DEBUG [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1668): Closing ce860946aa8cd08494bb77afea49e9de, disabling compactions & flushes 2023-10-02T12:30:13,479 INFO [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. 2023-10-02T12:30:13,479 DEBUG [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. 2023-10-02T12:30:13,479 DEBUG [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. after waiting 0 ms 2023-10-02T12:30:13,480 DEBUG [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. 2023-10-02T12:30:13,480 INFO [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. 2023-10-02T12:30:13,480 DEBUG [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1622): Region close journal for ce860946aa8cd08494bb77afea49e9de: 2023-10-02T12:30:13,481 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696249813481"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249813481"}]},"ts":"1696249813481"} 2023-10-02T12:30:13,484 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:30:13,485 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962497916551","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249813485"}]},"ts":"1696249813485"} 2023-10-02T12:30:13,487 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962497916551, state=ENABLING in hbase:meta 2023-10-02T12:30:13,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-10-02T12:30:13,498 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16962497916551, region=ce860946aa8cd08494bb77afea49e9de, ASSIGN}] 2023-10-02T12:30:13,501 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16962497916551, region=ce860946aa8cd08494bb77afea49e9de, ASSIGN 2023-10-02T12:30:13,502 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16962497916551, region=ce860946aa8cd08494bb77afea49e9de, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38255,1696249782114; forceNewPlan=false, retain=false 2023-10-02T12:30:13,654 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=42 updating hbase:meta row=ce860946aa8cd08494bb77afea49e9de, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:13,657 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; OpenRegionProcedure ce860946aa8cd08494bb77afea49e9de, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:30:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-10-02T12:30:13,816 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(130): Open ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. 2023-10-02T12:30:13,816 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7254): Opening region: {ENCODED => ce860946aa8cd08494bb77afea49e9de, NAME => 'ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:30:13,817 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:30:13,817 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962497916551 ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:30:13,817 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(888): Instantiated ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:30:13,817 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7296): checking encryption for ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:30:13,817 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7299): checking classloading for ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:30:13,823 INFO [StoreOpener-ce860946aa8cd08494bb77afea49e9de-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:30:13,825 INFO [StoreOpener-ce860946aa8cd08494bb77afea49e9de-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ce860946aa8cd08494bb77afea49e9de columnFamilyName f 2023-10-02T12:30:13,825 DEBUG [StoreOpener-ce860946aa8cd08494bb77afea49e9de-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:30:13,826 INFO [StoreOpener-ce860946aa8cd08494bb77afea49e9de-1 {}] regionserver.HStore(324): Store=ce860946aa8cd08494bb77afea49e9de/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:30:13,829 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:30:13,830 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:30:13,838 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1076): writing seq id for ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:30:13,845 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:30:13,846 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): Opened ce860946aa8cd08494bb77afea49e9de; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9996752480, jitterRate=-0.06897987425327301}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:30:13,847 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(992): Region open journal for ce860946aa8cd08494bb77afea49e9de: 2023-10-02T12:30:13,848 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de., pid=43, masterSystemTime=1696249813810 2023-10-02T12:30:13,850 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. 2023-10-02T12:30:13,850 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(158): Opened ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. 2023-10-02T12:30:13,854 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=42 updating hbase:meta row=ce860946aa8cd08494bb77afea49e9de, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:13,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=43, resume processing ppid=42 2023-10-02T12:30:13,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, ppid=42, state=SUCCESS; OpenRegionProcedure ce860946aa8cd08494bb77afea49e9de, server=jenkins-hbase3.apache.org,38255,1696249782114 in 200 msec 2023-10-02T12:30:13,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=42, resume processing ppid=41 2023-10-02T12:30:13,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, ppid=41, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16962497916551, region=ce860946aa8cd08494bb77afea49e9de, ASSIGN in 362 msec 2023-10-02T12:30:13,865 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962497916551","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249813865"}]},"ts":"1696249813865"} 2023-10-02T12:30:13,868 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962497916551, state=ENABLED in hbase:meta 2023-10-02T12:30:13,870 DEBUG [PEWorker-3 {}] procedure.TruncateTableProcedure(145): truncate 'ns2:test-16962497916551' completed 2023-10-02T12:30:13,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, state=SUCCESS; TruncateTableProcedure (table=ns2:test-16962497916551 preserveSplits=true) in 495 msec 2023-10-02T12:30:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=41 2023-10-02T12:30:13,998 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$TableFuture(3677): Operation: TRUNCATE, Table Name: ns2:test-16962497916551, procId: 41 completed 2023-10-02T12:30:14,008 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:37823/backupUT/backup_1696249798432/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9 from hbase tables ns2:test-16962497916551 to tables ns2:test-16962497916551 2023-10-02T12:30:14,008 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-16962497916551 into ns2:test-16962497916551 2023-10-02T12:30:14,014 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-test-16962497916551-1696249814008 from hdfs://localhost:37823/backupUT/backup_1696249798432/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9 2023-10-02T12:30:14,019 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3e0e0fd4 to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:30:14,031 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@783683e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:14,034 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:30:14,037 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50848, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:30:14,058 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-10-02T12:30:14,058 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16962497916551 2023-10-02T12:30:14,066 DEBUG [hconnection-0x62d236cb-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:14,067 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48640, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:14,072 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16962497916551: 2023-10-02T12:30:14,073 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-10-02T12:30:14,073 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/partitions_aa6d6259-e5db-40e9-8b13-2ea3aa8adc1f 2023-10-02T12:30:14,540 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:14,541 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:14,542 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:14,543 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:14,865 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-10-02T12:30:14,867 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50860, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-10-02T12:30:15,691 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-742260936210271021.jar 2023-10-02T12:30:15,692 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:15,692 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:15,692 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:15,693 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:15,693 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:15,694 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:15,694 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:15,695 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:15,695 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:15,696 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:30:15,696 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:30:15,697 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:30:15,697 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:30:15,698 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:30:15,698 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:30:15,699 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:30:15,699 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:30:15,700 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:30:15,700 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:30:15,701 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:30:15,701 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:30:15,705 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:15,705 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:15,705 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:15,706 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:15,706 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:15,707 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:15,707 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:15,707 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16962497916551 2023-10-02T12:30:15,708 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:30:15,708 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3e0e0fd4 to 127.0.0.1:57768 2023-10-02T12:30:15,708 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:15,708 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-10-02T12:30:15,708 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-10-02T12:30:15,709 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:30:15,714 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:30:15,720 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-10-02T12:30:15,811 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249815772/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:30:15,811 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249815772/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:30:15,919 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:37823/backupUT/backup_1696249798432/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/58e28812c4354ed58ad888f86b1669f9/f/a8381b3810df47eb97848d2aa7f60a19 2023-10-02T12:30:15,926 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-10-02T12:30:16,077 DEBUG [pool-202-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0a3b50d6 to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:30:16,084 DEBUG [pool-202-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@202232bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:16,086 DEBUG [hconnection-0x1a56456c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:16,087 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48656, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:16,094 DEBUG [pool-202-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0a3b50d6 to 127.0.0.1:57768 2023-10-02T12:30:16,094 DEBUG [pool-202-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:16,094 DEBUG [pool-202-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-10-02T12:30:16,095 DEBUG [pool-202-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-10-02T12:30:16,110 DEBUG [pool-202-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-test-16962497916551-1696249814008/_temporary/0/_temporary/attempt_local1742241315_0003_r_000000_0/f/49ce87b64cb1469cb5fcf40f6c5fad01 is 37, key is row10/f:q1/1696249795545/Put/seqid=0 2023-10-02T12:30:16,113 WARN [Thread-797 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:38255] Specified favored nodes: [jenkins-hbase3.apache.org:38255] 2023-10-02T12:30:16,878 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-test-16962497916551-1696249814008 2023-10-02T12:30:16,880 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x424fc6c9 to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:30:16,887 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77045ce0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:16,889 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:16,891 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48672, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:16,900 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-test-16962497916551-1696249814008/_SUCCESS 2023-10-02T12:30:16,905 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:30:16,907 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50864, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:30:16,944 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-test-16962497916551-1696249814008/f/49ce87b64cb1469cb5fcf40f6c5fad01 first=Optional[row0] last=Optional[row98] 2023-10-02T12:30:16,972 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de., hostname=jenkins-hbase3.apache.org,38255,1696249782114, seqNum=2 for row with hfile group [{f,hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-test-16962497916551-1696249814008/f/49ce87b64cb1469cb5fcf40f6c5fad01}] 2023-10-02T12:30:16,993 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-test-16962497916551-1696249814008/f/49ce87b64cb1469cb5fcf40f6c5fad01 for inclusion in ce860946aa8cd08494bb77afea49e9de/f 2023-10-02T12:30:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-10-02T12:30:17,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(625): Region bounds: first= last= 2023-10-02T12:30:17,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HRegion(2520): Flush status journal for ce860946aa8cd08494bb77afea49e9de: 2023-10-02T12:30:17,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-test-16962497916551-1696249814008/f/49ce87b64cb1469cb5fcf40f6c5fad01 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__ns2_test-16962497916551__s3intci99ddul77i136likcd0j2u1209544cuoof8918bfi972918vfc3ig6rsvj/f/49ce87b64cb1469cb5fcf40f6c5fad01 2023-10-02T12:30:17,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__ns2_test-16962497916551__s3intci99ddul77i136likcd0j2u1209544cuoof8918bfi972918vfc3ig6rsvj/f/49ce87b64cb1469cb5fcf40f6c5fad01 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_ 2023-10-02T12:30:17,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x49615fdf to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:30:17,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c240c0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:17,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:30:17,023 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50866, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:30:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:17,029 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48684, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(428): write bulk load descriptor to backup ns2:test-16962497916551 with 1 entries 2023-10-02T12:30:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(1602): writing raw bulk path hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_ for ns2:test-16962497916551 ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:30:17,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(435): written 1 rows for bulk load of ns2:test-16962497916551 2023-10-02T12:30:17,054 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:30:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x49615fdf to 127.0.0.1:57768 2023-10-02T12:30:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:17,057 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__ns2_test-16962497916551__s3intci99ddul77i136likcd0j2u1209544cuoof8918bfi972918vfc3ig6rsvj/f/49ce87b64cb1469cb5fcf40f6c5fad01 into ce860946aa8cd08494bb77afea49e9de/f as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_ - updating store file list. 2023-10-02T12:30:17,065 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_ into ce860946aa8cd08494bb77afea49e9de/f 2023-10-02T12:30:17,065 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__ns2_test-16962497916551__s3intci99ddul77i136likcd0j2u1209544cuoof8918bfi972918vfc3ig6rsvj/f/49ce87b64cb1469cb5fcf40f6c5fad01 into ce860946aa8cd08494bb77afea49e9de/f (new location: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_) 2023-10-02T12:30:17,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__ns2_test-16962497916551__s3intci99ddul77i136likcd0j2u1209544cuoof8918bfi972918vfc3ig6rsvj/f/49ce87b64cb1469cb5fcf40f6c5fad01 2023-10-02T12:30:17,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7e9b73df to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:30:17,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fc6f6b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:30:17,085 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50878, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-10-02T12:30:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:17,090 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48694, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-10-02T12:30:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup ns2:test-16962497916551 with 1 entries 2023-10-02T12:30:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(1523): writing done bulk path hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_ for ns2:test-16962497916551 ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:30:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(414): written 1 rows for bulk load of ns2:test-16962497916551 2023-10-02T12:30:17,114 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:30:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7e9b73df to 127.0.0.1:57768 2023-10-02T12:30:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:17,128 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:30:17,128 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x424fc6c9 to 127.0.0.1:57768 2023-10-02T12:30:17,128 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:17,129 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-10-02T12:30:17,129 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-10-02T12:30:17,129 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(233): Backup: backup_1696249798432 hdfs://localhost:37823/backupUT/backup_1696249798432/ns2/test-16962497916551/ 2023-10-02T12:30:17,130 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249798432/.backup.manifest 2023-10-02T12:30:17,134 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249798432/.backup.manifest 2023-10-02T12:30:17,134 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-16962497916552' to 'ns3:test-16962497916552' from full backup image hdfs://localhost:37823/backupUT/backup_1696249798432/ns3/test-16962497916552 2023-10-02T12:30:17,142 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:37823/backupUT/backup_1696249798432/ns3/test-16962497916552/archive/data/ns3/test-16962497916552 does not exists 2023-10-02T12:30:17,142 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-16962497916552, will only create table 2023-10-02T12:30:17,144 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(476): Truncating exising target table 'ns3:test-16962497916552', preserving region splits 2023-10-02T12:30:17,145 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$18(967): Started disable of ns3:test-16962497916552 2023-10-02T12:30:17,145 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable ns3:test-16962497916552 2023-10-02T12:30:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=44, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:test-16962497916552 2023-10-02T12:30:17,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:30:17,150 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962497916552","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249817150"}]},"ts":"1696249817150"} 2023-10-02T12:30:17,151 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962497916552, state=DISABLING in hbase:meta 2023-10-02T12:30:17,153 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set ns3:test-16962497916552 to state=DISABLING 2023-10-02T12:30:17,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-16962497916552, region=098a211186378b9b8d3dac0c285e5bf2, UNASSIGN}] 2023-10-02T12:30:17,156 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-16962497916552, region=098a211186378b9b8d3dac0c285e5bf2, UNASSIGN 2023-10-02T12:30:17,156 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=098a211186378b9b8d3dac0c285e5bf2, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:17,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; CloseRegionProcedure 098a211186378b9b8d3dac0c285e5bf2, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:30:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:30:17,311 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(119): Close 098a211186378b9b8d3dac0c285e5bf2 2023-10-02T12:30:17,311 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1668): Closing 098a211186378b9b8d3dac0c285e5bf2, disabling compactions & flushes 2023-10-02T12:30:17,311 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1690): Closing region ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2. 2023-10-02T12:30:17,311 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2. 2023-10-02T12:30:17,311 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2. after waiting 0 ms 2023-10-02T12:30:17,312 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2. 2023-10-02T12:30:17,320 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/098a211186378b9b8d3dac0c285e5bf2/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2023-10-02T12:30:17,321 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:30:17,322 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1909): Closed ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2. 2023-10-02T12:30:17,322 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1622): Region close journal for 098a211186378b9b8d3dac0c285e5bf2: 2023-10-02T12:30:17,325 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(163): Closed 098a211186378b9b8d3dac0c285e5bf2 2023-10-02T12:30:17,325 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=098a211186378b9b8d3dac0c285e5bf2, regionState=CLOSED 2023-10-02T12:30:17,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=46, resume processing ppid=45 2023-10-02T12:30:17,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, ppid=45, state=SUCCESS; CloseRegionProcedure 098a211186378b9b8d3dac0c285e5bf2, server=jenkins-hbase3.apache.org,38255,1696249782114 in 168 msec 2023-10-02T12:30:17,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=45, resume processing ppid=44 2023-10-02T12:30:17,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, ppid=44, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16962497916552, region=098a211186378b9b8d3dac0c285e5bf2, UNASSIGN in 176 msec 2023-10-02T12:30:17,333 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962497916552","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249817333"}]},"ts":"1696249817333"} 2023-10-02T12:30:17,335 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962497916552, state=DISABLED in hbase:meta 2023-10-02T12:30:17,337 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set ns3:test-16962497916552 to state=DISABLED 2023-10-02T12:30:17,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, state=SUCCESS; DisableTableProcedure table=ns3:test-16962497916552 in 193 msec 2023-10-02T12:30:17,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=44 2023-10-02T12:30:17,454 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: ns3:test-16962497916552, procId: 44 completed 2023-10-02T12:30:17,455 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$16(847): Started truncating ns3:test-16962497916552 2023-10-02T12:30:17,455 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.HMaster$6(2480): Client=jenkins//172.31.12.81 truncate ns3:test-16962497916552 2023-10-02T12:30:17,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=47, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns3:test-16962497916552 preserveSplits=true) 2023-10-02T12:30:17,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-10-02T12:30:17,458 DEBUG [PEWorker-5 {}] procedure.TruncateTableProcedure(87): waiting for 'ns3:test-16962497916552' regions in transition 2023-10-02T12:30:17,462 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/098a211186378b9b8d3dac0c285e5bf2 2023-10-02T12:30:17,465 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/098a211186378b9b8d3dac0c285e5bf2/f, FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/098a211186378b9b8d3dac0c285e5bf2/recovered.edits] 2023-10-02T12:30:17,474 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/098a211186378b9b8d3dac0c285e5bf2/recovered.edits/5.seqid to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/archive/data/ns3/test-16962497916552/098a211186378b9b8d3dac0c285e5bf2/recovered.edits/5.seqid 2023-10-02T12:30:17,474 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/098a211186378b9b8d3dac0c285e5bf2 2023-10-02T12:30:17,474 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived ns3:test-16962497916552 regions 2023-10-02T12:30:17,480 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:test-16962497916552 from hbase:meta 2023-10-02T12:30:17,484 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:test-16962497916552' descriptor. 2023-10-02T12:30:17,485 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:test-16962497916552' from region states. 2023-10-02T12:30:17,485 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696249817485"}]},"ts":"9223372036854775807"} 2023-10-02T12:30:17,490 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:30:17,490 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 098a211186378b9b8d3dac0c285e5bf2, NAME => 'ns3:test-16962497916552,,1696249795622.098a211186378b9b8d3dac0c285e5bf2.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:30:17,490 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:test-16962497916552' as deleted. 2023-10-02T12:30:17,490 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-16962497916552","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1696249817490"}]},"ts":"9223372036854775807"} 2023-10-02T12:30:17,492 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:test-16962497916552 state from META 2023-10-02T12:30:17,507 INFO [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 9d011caaefe2252a8728679a2f592847, NAME => 'ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16962497916552', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:30:17,514 DEBUG [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:30:17,514 DEBUG [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1668): Closing 9d011caaefe2252a8728679a2f592847, disabling compactions & flushes 2023-10-02T12:30:17,514 INFO [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. 2023-10-02T12:30:17,514 DEBUG [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. 2023-10-02T12:30:17,514 DEBUG [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. after waiting 0 ms 2023-10-02T12:30:17,514 DEBUG [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. 2023-10-02T12:30:17,514 INFO [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. 2023-10-02T12:30:17,514 DEBUG [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1622): Region close journal for 9d011caaefe2252a8728679a2f592847: 2023-10-02T12:30:17,516 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696249817516"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249817516"}]},"ts":"1696249817516"} 2023-10-02T12:30:17,518 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:30:17,519 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962497916552","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249817519"}]},"ts":"1696249817519"} 2023-10-02T12:30:17,521 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962497916552, state=ENABLING in hbase:meta 2023-10-02T12:30:17,526 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16962497916552, region=9d011caaefe2252a8728679a2f592847, ASSIGN}] 2023-10-02T12:30:17,528 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16962497916552, region=9d011caaefe2252a8728679a2f592847, ASSIGN 2023-10-02T12:30:17,529 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16962497916552, region=9d011caaefe2252a8728679a2f592847, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38255,1696249782114; forceNewPlan=false, retain=false 2023-10-02T12:30:17,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-10-02T12:30:17,681 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=9d011caaefe2252a8728679a2f592847, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:17,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE; OpenRegionProcedure 9d011caaefe2252a8728679a2f592847, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:30:17,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-10-02T12:30:17,840 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(130): Open ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. 2023-10-02T12:30:17,840 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7254): Opening region: {ENCODED => 9d011caaefe2252a8728679a2f592847, NAME => 'ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:30:17,840 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:30:17,841 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962497916552 9d011caaefe2252a8728679a2f592847 2023-10-02T12:30:17,841 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(888): Instantiated ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:30:17,841 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7296): checking encryption for 9d011caaefe2252a8728679a2f592847 2023-10-02T12:30:17,841 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7299): checking classloading for 9d011caaefe2252a8728679a2f592847 2023-10-02T12:30:17,843 INFO [StoreOpener-9d011caaefe2252a8728679a2f592847-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 9d011caaefe2252a8728679a2f592847 2023-10-02T12:30:17,845 INFO [StoreOpener-9d011caaefe2252a8728679a2f592847-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d011caaefe2252a8728679a2f592847 columnFamilyName f 2023-10-02T12:30:17,845 DEBUG [StoreOpener-9d011caaefe2252a8728679a2f592847-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:30:17,846 INFO [StoreOpener-9d011caaefe2252a8728679a2f592847-1 {}] regionserver.HStore(324): Store=9d011caaefe2252a8728679a2f592847/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:30:17,847 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/9d011caaefe2252a8728679a2f592847 2023-10-02T12:30:17,847 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/9d011caaefe2252a8728679a2f592847 2023-10-02T12:30:17,850 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1076): writing seq id for 9d011caaefe2252a8728679a2f592847 2023-10-02T12:30:17,853 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/9d011caaefe2252a8728679a2f592847/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:30:17,853 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): Opened 9d011caaefe2252a8728679a2f592847; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11673504960, jitterRate=0.08717986941337585}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:30:17,854 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(992): Region open journal for 9d011caaefe2252a8728679a2f592847: 2023-10-02T12:30:17,855 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847., pid=49, masterSystemTime=1696249817835 2023-10-02T12:30:17,857 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. 2023-10-02T12:30:17,857 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(158): Opened ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. 2023-10-02T12:30:17,858 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=9d011caaefe2252a8728679a2f592847, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:17,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=49, resume processing ppid=48 2023-10-02T12:30:17,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, ppid=48, state=SUCCESS; OpenRegionProcedure 9d011caaefe2252a8728679a2f592847, server=jenkins-hbase3.apache.org,38255,1696249782114 in 176 msec 2023-10-02T12:30:17,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=48, resume processing ppid=47 2023-10-02T12:30:17,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, ppid=47, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16962497916552, region=9d011caaefe2252a8728679a2f592847, ASSIGN in 335 msec 2023-10-02T12:30:17,863 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962497916552","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249817863"}]},"ts":"1696249817863"} 2023-10-02T12:30:17,865 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962497916552, state=ENABLED in hbase:meta 2023-10-02T12:30:17,872 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(145): truncate 'ns3:test-16962497916552' completed 2023-10-02T12:30:17,874 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, state=SUCCESS; TruncateTableProcedure (table=ns3:test-16962497916552 preserveSplits=true) in 417 msec 2023-10-02T12:30:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=47 2023-10-02T12:30:18,065 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$TableFuture(3677): Operation: TRUNCATE, Table Name: ns3:test-16962497916552, procId: 47 completed 2023-10-02T12:30:18,069 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-10-02T12:30:18,069 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(233): Backup: backup_1696249798432 hdfs://localhost:37823/backupUT/backup_1696249798432/ns3/test-16962497916552/ 2023-10-02T12:30:18,069 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-10-02T12:30:18,089 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleOverwrite Thread=598 (was 583) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/cluster_f5efbd98-923a-ea34-67ad-be7e4cbb4c4b/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x712cc456-shared-pool-13 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/cluster_f5efbd98-923a-ea34-67ad-be7e4cbb4c4b/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-2025135827-172.31.12.81-1696249775150:blk_1073741856_1032, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x712cc456-shared-pool-16 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-2025135827-172.31.12.81-1696249775150:blk_1073741855_1031, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x712cc456-shared-pool-12 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x712cc456-shared-pool-11 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2000333002_16 at /127.0.0.1:56336 [Receiving block BP-2025135827-172.31.12.81-1696249775150:blk_1073741856_1032] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x3eb3393a-shared-pool-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2000333002_16 at /127.0.0.1:56324 [Receiving block BP-2025135827-172.31.12.81-1696249775150:blk_1073741855_1031] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1482744570_16 at /127.0.0.1:56388 [Waiting for operation #28] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2000333002_16 at /127.0.0.1:56452 [Waiting for operation #24] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x712cc456-shared-pool-15 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x712cc456-shared-pool-14 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=675 (was 660) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=639 (was 576) - SystemLoadAverage LEAK? -, ProcessCount=180 (was 180), AvailableMemoryMB=5445 (was 5218) - AvailableMemoryMB LEAK? - 2023-10-02T12:30:18,089 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=598 is superior to 500 2023-10-02T12:30:18,106 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleOverwrite Thread=598, OpenFileDescriptor=675, MaxFileDescriptor=60000, SystemLoadAverage=639, ProcessCount=180, AvailableMemoryMB=5445 2023-10-02T12:30:18,106 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=598 is superior to 500 2023-10-02T12:30:18,106 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestFullRestore(173): test full restore on a single table empty table 2023-10-02T12:30:18,108 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4c44f83e to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:30:18,114 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16136ef6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:18,118 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:18,120 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48706, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:18,122 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:30:18,126 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50892, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:30:18,144 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:30:18,152 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:30:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249818152 (current time:1696249818152). 2023-10-02T12:30:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:30:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:30:18,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x411b898a to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:30:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ad8eb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:18,166 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48716, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x411b898a to 127.0.0.1:57768 2023-10-02T12:30:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:30:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:30:18,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-10-02T12:30:18,171 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:30:18,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:30:18,172 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:30:18,175 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:30:18,178 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:30:18,191 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:30:18,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987}] 2023-10-02T12:30:18,193 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:30:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:30:18,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2023-10-02T12:30:18,345 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:30:18,346 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2819): Flushing 7ec576d712d0f3b0c0592584b88ce987 2/2 column families, dataSize=1.51 KB heapSize=2.81 KB 2023-10-02T12:30:18,366 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/be4f942a12804891a1f8bc2a12951f98 is 171, key is trslm:hdfs://localhost:37823/backupUT\x00ns2:test-16962497916551/meta:log-roll-map/1696249812682/Put/seqid=0 2023-10-02T12:30:18,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:30:18,774 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=792 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/be4f942a12804891a1f8bc2a12951f98 2023-10-02T12:30:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:30:18,786 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/96b7c481d10942d6a12bd6cb944c1f6c is 458, key is session:backup_1696249798432/session:context/1696249812826/Put/seqid=0 2023-10-02T12:30:18,796 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=754 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/96b7c481d10942d6a12bd6cb944c1f6c 2023-10-02T12:30:18,812 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/be4f942a12804891a1f8bc2a12951f98 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98 2023-10-02T12:30:18,821 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98, entries=6, sequenceid=17, filesize=5.8 K 2023-10-02T12:30:18,822 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/96b7c481d10942d6a12bd6cb944c1f6c as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c 2023-10-02T12:30:18,831 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c, entries=2, sequenceid=17, filesize=5.4 K 2023-10-02T12:30:18,832 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(3022): Finished flush of dataSize ~1.51 KB/1546, heapSize ~2.78 KB/2848, currentSize=0 B/0 for 7ec576d712d0f3b0c0592584b88ce987 in 486ms, sequenceid=17, compaction requested=false 2023-10-02T12:30:18,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2520): Flush status journal for 7ec576d712d0f3b0c0592584b88ce987: 2023-10-02T12:30:18,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. for snapshot_backup_system completed. 2023-10-02T12:30:18,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:30:18,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:30:18,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98] hfiles 2023-10-02T12:30:18,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98 for snapshot=snapshot_backup_system 2023-10-02T12:30:18,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c] hfiles 2023-10-02T12:30:18,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0 for snapshot=snapshot_backup_system 2023-10-02T12:30:18,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c for snapshot=snapshot_backup_system 2023-10-02T12:30:18,840 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:30:18,840 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=51 2023-10-02T12:30:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=51 2023-10-02T12:30:18,841 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:30:18,841 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:30:18,844 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:30:18,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-10-02T12:30:18,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 in 654 msec 2023-10-02T12:30:18,849 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:30:18,851 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:30:18,851 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:30:18,852 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:30:18,853 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:30:18,903 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:30:18,921 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:30:18,922 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:30:18,924 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:30:18,924 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-10-02T12:30:18,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 756 msec 2023-10-02T12:30:19,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=50 2023-10-02T12:30:19,278 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 50 completed 2023-10-02T12:30:19,279 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(115): Backup backup_1696249818114 started at 1696249819279. 2023-10-02T12:30:19,280 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(119): Backup session backup_1696249818114 has been started. 2023-10-02T12:30:19,286 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:30:19,287 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:30:19,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:30:19,287 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:30:19,287 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:30:19,288 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:30:19,288 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:30:19,289 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:19,289 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:19,291 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:19,291 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:30:19,291 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:30:19,291 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:30:19,292 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:19,292 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:30:19,292 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:19,292 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:19,293 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:30:19,293 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:19,293 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:30:19,293 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:30:19,293 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:30:19,293 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:30:19,293 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:30:19,294 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:30:19,294 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:30:19,294 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:30:19,297 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:19,297 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-10-02T12:30:19,297 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:30:19,297 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:19,298 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:19,298 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:30:19,298 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:30:19,298 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:30:19,298 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:30:19,299 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:19,299 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:19,299 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:30:19,300 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:30:19,300 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@4c6ba6a1[Count = 0] remaining members to acquire global barrier 2023-10-02T12:30:19,300 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:30:19,300 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:19,301 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:19,301 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:19,301 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:19,301 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:30:19,301 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:19,302 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:30:19,302 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:30:19,302 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:19,302 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696249802681 highest: 1696249802681 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:19,302 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114:(num 1696249802681) roll requested 2023-10-02T12:30:19,314 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249819302, exclude list is [], retry=0 2023-10-02T12:30:19,319 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:30:19,321 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249802681 with entries=19, filesize=6.26 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249819302 2023-10-02T12:30:19,321 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:30:19,321 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249802681 is not closed yet, will try archiving it next time 2023-10-02T12:30:19,321 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta:.meta(num 1696249802720) roll requested 2023-10-02T12:30:19,334 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249819322.meta, exclude list is [], retry=0 2023-10-02T12:30:19,339 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:30:19,341 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249802720.meta with entries=24, filesize=6.26 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249819322.meta 2023-10-02T12:30:19,341 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:30:19,341 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249802720.meta is not closed yet, will try archiving it next time 2023-10-02T12:30:19,343 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-10-02T12:30:19,343 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696249819302 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:19,344 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249802720.meta not finished, retry = 0 2023-10-02T12:30:19,353 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:30:19,353 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,38255,1696249782114' in zk 2023-10-02T12:30:19,356 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:19,356 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:30:19,356 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:30:19,356 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:30:19,357 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:19,357 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:30:19,357 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:30:19,358 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:30:19,358 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:30:19,358 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:19,359 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:19,359 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:30:19,360 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:19,360 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:19,361 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,38255,1696249782114': 2023-10-02T12:30:19,361 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:30:19,361 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:30:19,361 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:30:19,361 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:30:19,361 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:30:19,361 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:30:19,363 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:30:19,363 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:30:19,364 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:30:19,364 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:19,364 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:30:19,364 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:30:19,364 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:19,364 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:30:19,365 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:30:19,365 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:30:19,365 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:19,365 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:30:19,366 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:19,366 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:19,366 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:30:19,367 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:19,367 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:19,367 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:19,368 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:30:19,368 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:19,369 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:19,373 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:30:19,373 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:19,373 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:30:19,373 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:30:19,373 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:30:19,373 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:19,373 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:19,373 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:30:19,373 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:19,373 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:30:19,373 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-10-02T12:30:19,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:30:19,373 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:19,374 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:19,374 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:30:19,374 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:30:19,374 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:30:19,374 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-10-02T12:30:19,374 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-10-02T12:30:19,475 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-10-02T12:30:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:30:19,480 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249819479_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } 2023-10-02T12:30:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249819480 (current time:1696249819480). 2023-10-02T12:30:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:30:19,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249819479_default_test-1696249791655 VERSION not specified, setting to 2 2023-10-02T12:30:19,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x73ac3440 to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:30:19,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@181fc4a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:19,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:19,493 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48728, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x73ac3440 to 127.0.0.1:57768 2023-10-02T12:30:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:19,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:30:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1696249819479_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } 2023-10-02T12:30:19,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249819479_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-10-02T12:30:19,497 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1696249819479_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:30:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:30:19,498 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1696249819479_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:30:19,501 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249819479_default_test-1696249791655 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:30:19,504 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1696249819479_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:30:19,510 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1696249819479_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:30:19,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure ec65563da527c2d4de1d128d9fb4f28f}] 2023-10-02T12:30:19,512 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure ec65563da527c2d4de1d128d9fb4f28f 2023-10-02T12:30:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:30:19,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2023-10-02T12:30:19,664 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f. 2023-10-02T12:30:19,665 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2819): Flushing ec65563da527c2d4de1d128d9fb4f28f 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-10-02T12:30:19,683 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f/.tmp/f/9d570c5d61e04e0bb70f21cbd68f28c7 is 37, key is row10/f:q1/1696249794327/Put/seqid=0 2023-10-02T12:30:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:30:20,091 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f/.tmp/f/9d570c5d61e04e0bb70f21cbd68f28c7 2023-10-02T12:30:20,100 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f/.tmp/f/9d570c5d61e04e0bb70f21cbd68f28c7 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f/f/9d570c5d61e04e0bb70f21cbd68f28c7 2023-10-02T12:30:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:30:20,107 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f/f/9d570c5d61e04e0bb70f21cbd68f28c7, entries=99, sequenceid=103, filesize=8.2 K 2023-10-02T12:30:20,108 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for ec65563da527c2d4de1d128d9fb4f28f in 443ms, sequenceid=103, compaction requested=false 2023-10-02T12:30:20,109 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2520): Flush status journal for ec65563da527c2d4de1d128d9fb4f28f: 2023-10-02T12:30:20,109 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f. for snapshot_1696249819479_default_test-1696249791655 completed. 2023-10-02T12:30:20,110 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(240): Storing 'test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f.' region-info for snapshot=snapshot_1696249819479_default_test-1696249791655 2023-10-02T12:30:20,110 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:30:20,110 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f/f/9d570c5d61e04e0bb70f21cbd68f28c7] hfiles 2023-10-02T12:30:20,110 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f/f/9d570c5d61e04e0bb70f21cbd68f28c7 for snapshot=snapshot_1696249819479_default_test-1696249791655 2023-10-02T12:30:20,517 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f. 2023-10-02T12:30:20,517 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=53 2023-10-02T12:30:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=53 2023-10-02T12:30:20,518 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249819479_default_test-1696249791655 on region ec65563da527c2d4de1d128d9fb4f28f 2023-10-02T12:30:20,519 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure ec65563da527c2d4de1d128d9fb4f28f 2023-10-02T12:30:20,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=53, resume processing ppid=52 2023-10-02T12:30:20,522 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1696249819479_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:30:20,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, ppid=52, state=SUCCESS; SnapshotRegionProcedure ec65563da527c2d4de1d128d9fb4f28f in 1.0100 sec 2023-10-02T12:30:20,523 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1696249819479_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:30:20,523 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1696249819479_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:30:20,524 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249819479_default_test-1696249791655 2023-10-02T12:30:20,524 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249819479_default_test-1696249791655 2023-10-02T12:30:20,538 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1696249819479_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:30:20,545 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1696249819479_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:30:20,545 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249819479_default_test-1696249791655 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249819479_default_test-1696249791655 2023-10-02T12:30:20,546 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1696249819479_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:30:20,547 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249819479_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-10-02T12:30:20,548 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1696249819479_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } in 1.0520 sec 2023-10-02T12:30:20,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=52 2023-10-02T12:30:20,605 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: default:test-1696249791655, procId: 52 completed 2023-10-02T12:30:20,606 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696249818114 2023-10-02T12:30:20,606 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:30:20,606 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-10-02T12:30:20,606 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249818114_test-1696249791655 2023-10-02T12:30:20,606 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249819479_default_test-1696249791655 to hdfs://localhost:37823/backupUT/backup_1696249818114/default/test-1696249791655/ 2023-10-02T12:30:20,606 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:30:20,633 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:30:20,633 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,40619,1696249778965_429384459_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37823/backupUT/backup_1696249818114/default/test-1696249791655, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37823/backupUT/backup_1696249818114/default/test-1696249791655/.hbase-snapshot/.tmp/snapshot_1696249819479_default_test-1696249791655 2023-10-02T12:30:20,633 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07, snapshotDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249819479_default_test-1696249791655. 2023-10-02T12:30:20,646 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249819479_default_test-1696249791655 to hdfs://localhost:37823/backupUT/backup_1696249818114/default/test-1696249791655/.hbase-snapshot/.tmp/snapshot_1696249819479_default_test-1696249791655 2023-10-02T12:30:21,069 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:21,069 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:21,070 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:21,071 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:22,262 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-8041812155458803092.jar 2023-10-02T12:30:22,263 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:22,263 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:22,264 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:22,265 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:22,266 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:22,266 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:22,267 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:22,268 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:22,268 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:22,270 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:30:22,271 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:30:22,271 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:30:22,272 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:30:22,273 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:30:22,273 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:30:22,274 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:30:22,274 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:30:22,275 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:30:22,276 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:30:22,277 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:30:22,277 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:30:22,278 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:22,279 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:22,280 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:22,280 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:22,281 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:22,282 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:22,283 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:22,334 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:30:22,375 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249819479_default_test-1696249791655' hfile list 2023-10-02T12:30:22,385 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-10-02T12:30:22,475 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249822428/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:30:22,476 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249822428/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:30:22,556 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:30:22,583 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f/f/9d570c5d61e04e0bb70f21cbd68f28c7 output=hdfs://localhost:37823/backupUT/backup_1696249818114/default/test-1696249791655/archive/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f/f/9d570c5d61e04e0bb70f21cbd68f28c7 2023-10-02T12:30:22,583 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 8.007M/sec 2023-10-02T12:30:23,533 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:30:23,535 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:30:23,545 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249819479_default_test-1696249791655 2023-10-02T12:30:23,545 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249819479_default_test-1696249791655 finished. 2023-10-02T12:30:23,545 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1034): test-1696249791655 2023-10-02T12:30:23,560 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249818114 2023-10-02T12:30:23,560 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:30:23,570 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37823/backupUT/backup_1696249818114/.backup.manifest 2023-10-02T12:30:23,570 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(358): Backup backup_1696249818114 finished: type=FULL,tablelist=test-1696249791655,targetRootDir=hdfs://localhost:37823/backupUT,startts=1696249819279,completets=1696249823560,bytescopied=0 2023-10-02T12:30:23,570 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:30:23,570 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249819479_default_test-1696249791655 2023-10-02T12:30:23,572 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249819479_default_test-1696249791655" 2023-10-02T12:30:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249819479_default_test-1696249791655 2023-10-02T12:30:23,577 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249819479_default_test-1696249791655 for backup backup_1696249818114 succeeded. 2023-10-02T12:30:23,577 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:30:23,585 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:30:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:30:23,592 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:30:23,594 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:30:23,604 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(376): Backup backup_1696249818114 completed. 2023-10-02T12:30:23,605 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:30:23,606 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4c44f83e to 127.0.0.1:57768 2023-10-02T12:30:23,606 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:23,628 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestFullRestore(178): backup complete 2023-10-02T12:30:23,632 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249818114/.backup.manifest 2023-10-02T12:30:23,636 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249818114/.backup.manifest 2023-10-02T12:30:23,637 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249818114/.backup.manifest 2023-10-02T12:30:23,640 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249818114/.backup.manifest 2023-10-02T12:30:23,640 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(148): Restoring 'test-1696249791655' to 'test-1696249791655' from full backup image hdfs://localhost:37823/backupUT/backup_1696249818114/default/test-1696249791655 2023-10-02T12:30:23,651 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(476): Truncating exising target table 'test-1696249791655', preserving region splits 2023-10-02T12:30:23,652 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$18(967): Started disable of test-1696249791655 2023-10-02T12:30:23,652 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable test-1696249791655 2023-10-02T12:30:23,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=test-1696249791655 2023-10-02T12:30:23,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-10-02T12:30:23,657 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696249791655","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249823657"}]},"ts":"1696249823657"} 2023-10-02T12:30:23,658 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696249791655, state=DISABLING in hbase:meta 2023-10-02T12:30:23,660 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set test-1696249791655 to state=DISABLING 2023-10-02T12:30:23,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1696249791655, region=ec65563da527c2d4de1d128d9fb4f28f, UNASSIGN}] 2023-10-02T12:30:23,663 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1696249791655, region=ec65563da527c2d4de1d128d9fb4f28f, UNASSIGN 2023-10-02T12:30:23,664 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=ec65563da527c2d4de1d128d9fb4f28f, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:23,666 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; CloseRegionProcedure ec65563da527c2d4de1d128d9fb4f28f, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:30:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-10-02T12:30:23,824 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(119): Close ec65563da527c2d4de1d128d9fb4f28f 2023-10-02T12:30:23,825 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1668): Closing ec65563da527c2d4de1d128d9fb4f28f, disabling compactions & flushes 2023-10-02T12:30:23,825 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1690): Closing region test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f. 2023-10-02T12:30:23,825 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f. 2023-10-02T12:30:23,825 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1778): Acquired close lock on test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f. after waiting 0 ms 2023-10-02T12:30:23,825 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1788): Updates disabled for region test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f. 2023-10-02T12:30:23,842 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-10-02T12:30:23,843 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:30:23,846 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1909): Closed test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f. 2023-10-02T12:30:23,846 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1622): Region close journal for ec65563da527c2d4de1d128d9fb4f28f: 2023-10-02T12:30:23,849 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=ec65563da527c2d4de1d128d9fb4f28f, regionState=CLOSED 2023-10-02T12:30:23,854 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(163): Closed ec65563da527c2d4de1d128d9fb4f28f 2023-10-02T12:30:23,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=56, resume processing ppid=55 2023-10-02T12:30:23,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=56, ppid=55, state=SUCCESS; CloseRegionProcedure ec65563da527c2d4de1d128d9fb4f28f, server=jenkins-hbase3.apache.org,38255,1696249782114 in 194 msec 2023-10-02T12:30:23,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=55, resume processing ppid=54 2023-10-02T12:30:23,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=test-1696249791655, region=ec65563da527c2d4de1d128d9fb4f28f, UNASSIGN in 205 msec 2023-10-02T12:30:23,871 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696249791655","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249823871"}]},"ts":"1696249823871"} 2023-10-02T12:30:23,875 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696249791655, state=DISABLED in hbase:meta 2023-10-02T12:30:23,883 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set test-1696249791655 to state=DISABLED 2023-10-02T12:30:23,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=54, state=SUCCESS; DisableTableProcedure table=test-1696249791655 in 232 msec 2023-10-02T12:30:23,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=54 2023-10-02T12:30:23,964 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$TableFuture(3677): Operation: DISABLE, Table Name: default:test-1696249791655, procId: 54 completed 2023-10-02T12:30:23,966 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$16(847): Started truncating test-1696249791655 2023-10-02T12:30:23,966 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$6(2480): Client=jenkins//172.31.12.81 truncate test-1696249791655 2023-10-02T12:30:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=57, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=test-1696249791655 preserveSplits=true) 2023-10-02T12:30:23,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-10-02T12:30:23,973 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(87): waiting for 'test-1696249791655' regions in transition 2023-10-02T12:30:23,980 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f 2023-10-02T12:30:23,991 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f/f, FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f/recovered.edits] 2023-10-02T12:30:24,005 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f/f/9d570c5d61e04e0bb70f21cbd68f28c7 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/archive/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f/f/9d570c5d61e04e0bb70f21cbd68f28c7 2023-10-02T12:30:24,011 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f/recovered.edits/106.seqid to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/archive/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f/recovered.edits/106.seqid 2023-10-02T12:30:24,012 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f 2023-10-02T12:30:24,012 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived test-1696249791655 regions 2023-10-02T12:30:24,023 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of test-1696249791655 from hbase:meta 2023-10-02T12:30:24,027 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'test-1696249791655' descriptor. 2023-10-02T12:30:24,028 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'test-1696249791655' from region states. 2023-10-02T12:30:24,028 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696249824028"}]},"ts":"9223372036854775807"} 2023-10-02T12:30:24,048 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:30:24,049 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => ec65563da527c2d4de1d128d9fb4f28f, NAME => 'test-1696249791655,,1696249792155.ec65563da527c2d4de1d128d9fb4f28f.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:30:24,049 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'test-1696249791655' as deleted. 2023-10-02T12:30:24,049 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1696249791655","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1696249824049"}]},"ts":"9223372036854775807"} 2023-10-02T12:30:24,057 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table test-1696249791655 state from META 2023-10-02T12:30:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-10-02T12:30:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-10-02T12:30:24,503 INFO [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => c8f99eaa6d1b90df2e9c811158390413, NAME => 'test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1696249791655', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:30:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-10-02T12:30:24,916 DEBUG [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(888): Instantiated test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:30:24,916 DEBUG [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1668): Closing c8f99eaa6d1b90df2e9c811158390413, disabling compactions & flushes 2023-10-02T12:30:24,916 INFO [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1690): Closing region test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. 2023-10-02T12:30:24,916 DEBUG [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. 2023-10-02T12:30:24,916 DEBUG [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. after waiting 0 ms 2023-10-02T12:30:24,916 DEBUG [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. 2023-10-02T12:30:24,916 INFO [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1909): Closed test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. 2023-10-02T12:30:24,916 DEBUG [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1622): Region close journal for c8f99eaa6d1b90df2e9c811158390413: 2023-10-02T12:30:24,918 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1696249824918"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249824918"}]},"ts":"1696249824918"} 2023-10-02T12:30:24,920 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:30:24,922 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696249791655","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249824922"}]},"ts":"1696249824922"} 2023-10-02T12:30:24,923 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696249791655, state=ENABLING in hbase:meta 2023-10-02T12:30:24,929 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1696249791655, region=c8f99eaa6d1b90df2e9c811158390413, ASSIGN}] 2023-10-02T12:30:24,931 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1696249791655, region=c8f99eaa6d1b90df2e9c811158390413, ASSIGN 2023-10-02T12:30:24,932 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1696249791655, region=c8f99eaa6d1b90df2e9c811158390413, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38255,1696249782114; forceNewPlan=false, retain=false 2023-10-02T12:30:25,088 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=c8f99eaa6d1b90df2e9c811158390413, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:25,092 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; OpenRegionProcedure c8f99eaa6d1b90df2e9c811158390413, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:30:25,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-10-02T12:30:25,264 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(130): Open test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. 2023-10-02T12:30:25,264 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7254): Opening region: {ENCODED => c8f99eaa6d1b90df2e9c811158390413, NAME => 'test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:30:25,265 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:30:25,265 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1696249791655 c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:25,265 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(888): Instantiated test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:30:25,265 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7296): checking encryption for c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:25,265 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7299): checking classloading for c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:25,274 INFO [StoreOpener-c8f99eaa6d1b90df2e9c811158390413-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:25,280 INFO [StoreOpener-c8f99eaa6d1b90df2e9c811158390413-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8f99eaa6d1b90df2e9c811158390413 columnFamilyName f 2023-10-02T12:30:25,280 DEBUG [StoreOpener-c8f99eaa6d1b90df2e9c811158390413-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:30:25,281 INFO [StoreOpener-c8f99eaa6d1b90df2e9c811158390413-1 {}] regionserver.HStore(324): Store=c8f99eaa6d1b90df2e9c811158390413/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:30:25,286 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:25,286 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:25,296 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1076): writing seq id for c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:25,338 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:30:25,339 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): Opened c8f99eaa6d1b90df2e9c811158390413; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10489445760, jitterRate=-0.023094236850738525}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:30:25,340 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(992): Region open journal for c8f99eaa6d1b90df2e9c811158390413: 2023-10-02T12:30:25,341 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413., pid=59, masterSystemTime=1696249825256 2023-10-02T12:30:25,355 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. 2023-10-02T12:30:25,356 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(158): Opened test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. 2023-10-02T12:30:25,358 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=c8f99eaa6d1b90df2e9c811158390413, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:25,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=59, resume processing ppid=58 2023-10-02T12:30:25,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=59, ppid=58, state=SUCCESS; OpenRegionProcedure c8f99eaa6d1b90df2e9c811158390413, server=jenkins-hbase3.apache.org,38255,1696249782114 in 268 msec 2023-10-02T12:30:25,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=58, resume processing ppid=57 2023-10-02T12:30:25,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=58, ppid=57, state=SUCCESS; TransitRegionStateProcedure table=test-1696249791655, region=c8f99eaa6d1b90df2e9c811158390413, ASSIGN in 434 msec 2023-10-02T12:30:25,365 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696249791655","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249825365"}]},"ts":"1696249825365"} 2023-10-02T12:30:25,371 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696249791655, state=ENABLED in hbase:meta 2023-10-02T12:30:25,373 DEBUG [PEWorker-4 {}] procedure.TruncateTableProcedure(145): truncate 'test-1696249791655' completed 2023-10-02T12:30:25,375 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=57, state=SUCCESS; TruncateTableProcedure (table=test-1696249791655 preserveSplits=true) in 1.4060 sec 2023-10-02T12:30:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=57 2023-10-02T12:30:26,112 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$TableFuture(3677): Operation: TRUNCATE, Table Name: default:test-1696249791655, procId: 57 completed 2023-10-02T12:30:26,118 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:37823/backupUT/backup_1696249818114/default/test-1696249791655/archive/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f from hbase tables test-1696249791655 to tables test-1696249791655 2023-10-02T12:30:26,118 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1696249791655 into test-1696249791655 2023-10-02T12:30:26,123 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-test-1696249791655-1696249826118 from hdfs://localhost:37823/backupUT/backup_1696249818114/default/test-1696249791655/archive/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f 2023-10-02T12:30:26,125 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x24db51ab to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:30:26,138 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@721df5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:26,140 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:30:26,146 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38132, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:30:26,151 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-10-02T12:30:26,151 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1696249791655 2023-10-02T12:30:26,155 DEBUG [hconnection-0x6a634f1b-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:26,159 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49340, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:26,161 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1696249791655: 2023-10-02T12:30:26,161 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-10-02T12:30:26,161 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/partitions_b8716376-15f5-4458-8df2-1967df8eb4e9 2023-10-02T12:30:26,579 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:26,579 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:26,580 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:26,580 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:26,946 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-10-02T12:30:26,946 INFO [master/jenkins-hbase3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-10-02T12:30:27,770 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-199119998736369710.jar 2023-10-02T12:30:27,771 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:27,771 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:27,772 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:27,773 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:27,774 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:27,774 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:27,775 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:27,775 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:27,776 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:27,777 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:30:27,778 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:30:27,779 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:30:27,779 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:30:27,780 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:30:27,781 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:30:27,781 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:30:27,782 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:30:27,782 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:30:27,783 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:30:27,784 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:30:27,784 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:30:27,785 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:27,786 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:27,786 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:27,787 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:27,787 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:27,788 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:27,789 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:27,789 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: test-1696249791655 2023-10-02T12:30:27,789 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:30:27,789 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x24db51ab to 127.0.0.1:57768 2023-10-02T12:30:27,790 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:27,790 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-10-02T12:30:27,790 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-10-02T12:30:27,791 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:30:27,816 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:30:27,843 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-10-02T12:30:28,113 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249828014/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:30:28,113 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249828014/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:30:28,276 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:37823/backupUT/backup_1696249818114/default/test-1696249791655/archive/data/default/test-1696249791655/ec65563da527c2d4de1d128d9fb4f28f/f/9d570c5d61e04e0bb70f21cbd68f28c7 2023-10-02T12:30:28,285 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-10-02T12:30:28,338 DEBUG [pool-236-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x266b5cc0 to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:30:28,358 DEBUG [pool-236-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@505be8b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:28,362 DEBUG [hconnection-0x7799f530-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:28,366 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49354, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:28,373 DEBUG [pool-236-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x266b5cc0 to 127.0.0.1:57768 2023-10-02T12:30:28,373 DEBUG [pool-236-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:28,373 DEBUG [pool-236-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-10-02T12:30:28,373 DEBUG [pool-236-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-10-02T12:30:28,389 DEBUG [pool-236-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-test-1696249791655-1696249826118/_temporary/0/_temporary/attempt_local1770280399_0005_r_000000_0/f/19081298c1d14c34b2b1a8c7be50835d is 37, key is row10/f:q1/1696249794327/Put/seqid=0 2023-10-02T12:30:28,393 WARN [Thread-951 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:38255] Specified favored nodes: [jenkins-hbase3.apache.org:38255] 2023-10-02T12:30:29,202 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-test-1696249791655-1696249826118 2023-10-02T12:30:29,203 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x57069fcf to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:30:29,248 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f80d34a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:29,250 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:29,252 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49358, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:29,260 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-test-1696249791655-1696249826118/_SUCCESS 2023-10-02T12:30:29,270 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:30:29,272 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38138, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:30:29,292 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-test-1696249791655-1696249826118/f/19081298c1d14c34b2b1a8c7be50835d first=Optional[row0] last=Optional[row98] 2023-10-02T12:30:29,294 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413., hostname=jenkins-hbase3.apache.org,38255,1696249782114, seqNum=2 for row with hfile group [{f,hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-test-1696249791655-1696249826118/f/19081298c1d14c34b2b1a8c7be50835d}] 2023-10-02T12:30:29,311 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-test-1696249791655-1696249826118/f/19081298c1d14c34b2b1a8c7be50835d for inclusion in c8f99eaa6d1b90df2e9c811158390413/f 2023-10-02T12:30:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-10-02T12:30:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(625): Region bounds: first= last= 2023-10-02T12:30:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HRegion(2520): Flush status journal for c8f99eaa6d1b90df2e9c811158390413: 2023-10-02T12:30:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-test-1696249791655-1696249826118/f/19081298c1d14c34b2b1a8c7be50835d to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__test-1696249791655__au6me4rqg3hlbab17ac27hemsdk2r1l2cd7bosl9lfoccjibh9hm0g74vos2pb6b/f/19081298c1d14c34b2b1a8c7be50835d 2023-10-02T12:30:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__test-1696249791655__au6me4rqg3hlbab17ac27hemsdk2r1l2cd7bosl9lfoccjibh9hm0g74vos2pb6b/f/19081298c1d14c34b2b1a8c7be50835d as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ 2023-10-02T12:30:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6a9153e9 to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:30:29,348 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:30:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eeebadc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:30:29,427 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38154, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:30:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:29,437 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49372, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(428): write bulk load descriptor to backup test-1696249791655 with 1 entries 2023-10-02T12:30:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(1602): writing raw bulk path hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ for test-1696249791655 c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(435): written 1 rows for bulk load of test-1696249791655 2023-10-02T12:30:29,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:30:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6a9153e9 to 127.0.0.1:57768 2023-10-02T12:30:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:29,495 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__test-1696249791655__au6me4rqg3hlbab17ac27hemsdk2r1l2cd7bosl9lfoccjibh9hm0g74vos2pb6b/f/19081298c1d14c34b2b1a8c7be50835d into c8f99eaa6d1b90df2e9c811158390413/f as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ - updating store file list. 2023-10-02T12:30:29,505 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ into c8f99eaa6d1b90df2e9c811158390413/f 2023-10-02T12:30:29,505 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__test-1696249791655__au6me4rqg3hlbab17ac27hemsdk2r1l2cd7bosl9lfoccjibh9hm0g74vos2pb6b/f/19081298c1d14c34b2b1a8c7be50835d into c8f99eaa6d1b90df2e9c811158390413/f (new location: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_) 2023-10-02T12:30:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__test-1696249791655__au6me4rqg3hlbab17ac27hemsdk2r1l2cd7bosl9lfoccjibh9hm0g74vos2pb6b/f/19081298c1d14c34b2b1a8c7be50835d 2023-10-02T12:30:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x33b9485a to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:30:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@373dda6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:30:29,626 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38166, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-10-02T12:30:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:29,642 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49384, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-10-02T12:30:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup test-1696249791655 with 1 entries 2023-10-02T12:30:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(1523): writing done bulk path hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ for test-1696249791655 c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(414): written 1 rows for bulk load of test-1696249791655 2023-10-02T12:30:29,677 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:30:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x33b9485a to 127.0.0.1:57768 2023-10-02T12:30:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:29,684 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:30:29,685 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x57069fcf to 127.0.0.1:57768 2023-10-02T12:30:29,685 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:29,686 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-10-02T12:30:29,686 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-10-02T12:30:29,686 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(233): Backup: backup_1696249818114 hdfs://localhost:37823/backupUT/backup_1696249818114/default/test-1696249791655/ 2023-10-02T12:30:29,686 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-10-02T12:30:29,714 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleOverwrite Thread=600 (was 598) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57768@0x57069fcf-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: hconnection-0x712cc456-shared-pool-23 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1482744570_16 at /127.0.0.1:55352 [Waiting for operation #11] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57768@0x33b9485a-SendThread(127.0.0.1:57768) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:228) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57768@0x57069fcf-SendThread(127.0.0.1:57768) sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:345) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1223) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57768@0x33b9485a java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$482/273044086.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57768@0x57069fcf org.apache.zookeeper.ClientCnxn.queuePacket(ClientCnxn.java:1604) org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1520) org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1512) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1481) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$482/273044086.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2000333002_16 at /127.0.0.1:56976 [Receiving block BP-2025135827-172.31.12.81-1696249775150:blk_1073741882_1058] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-2025135827-172.31.12.81-1696249775150:blk_1073741883_1059, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-3 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57768@0x33b9485a-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: PacketResponder: BP-2025135827-172.31.12.81-1696249775150:blk_1073741882_1058, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2000333002_16 at /127.0.0.1:56986 [Receiving block BP-2025135827-172.31.12.81-1696249775150:blk_1073741883_1059] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x712cc456-shared-pool-22 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=695 (was 675) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=672 (was 639) - SystemLoadAverage LEAK? -, ProcessCount=181 (was 180) - ProcessCount LEAK? -, AvailableMemoryMB=4371 (was 5445) 2023-10-02T12:30:29,715 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=600 is superior to 500 2023-10-02T12:30:29,744 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleDNECommand Thread=599, OpenFileDescriptor=695, MaxFileDescriptor=60000, SystemLoadAverage=672, ProcessCount=181, AvailableMemoryMB=4369 2023-10-02T12:30:29,745 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=599 is superior to 500 2023-10-02T12:30:29,745 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestFullRestore(280): test restore fails on a single table that does not exist: command-line 2023-10-02T12:30:29,747 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5fda4e0c to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:30:29,777 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42d2a79a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:29,800 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:29,801 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49390, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:29,806 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:30:29,808 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:38168, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:30:29,853 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:30:29,868 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:30:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249829868 (current time:1696249829868). 2023-10-02T12:30:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:30:29,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:30:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0ee7af11 to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:30:29,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fbca25c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:29,897 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49394, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0ee7af11 to 127.0.0.1:57768 2023-10-02T12:30:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:29,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:30:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:30:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2023-10-02T12:30:29,902 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:30:29,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-10-02T12:30:29,903 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:30:29,907 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:30:29,910 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:30:30,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-10-02T12:30:30,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-10-02T12:30:30,319 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:30:30,319 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987}] 2023-10-02T12:30:30,323 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:30:30,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2023-10-02T12:30:30,476 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:30:30,476 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2819): Flushing 7ec576d712d0f3b0c0592584b88ce987 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-10-02T12:30:30,482 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/eeec1eb03cf84adab876e360f800c255 is 169, key is trslm:hdfs://localhost:37823/backupUT\x00test-1696249791655/meta:log-roll-map/1696249823555/Put/seqid=0 2023-10-02T12:30:30,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-10-02T12:30:30,888 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/eeec1eb03cf84adab876e360f800c255 2023-10-02T12:30:30,902 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/5fb18f8bc21a4dc1b98c0af588fef229 is 452, key is session:backup_1696249818114/session:context/1696249823593/Put/seqid=0 2023-10-02T12:30:30,909 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/5fb18f8bc21a4dc1b98c0af588fef229 2023-10-02T12:30:30,918 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/eeec1eb03cf84adab876e360f800c255 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255 2023-10-02T12:30:30,931 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255, entries=4, sequenceid=28, filesize=5.5 K 2023-10-02T12:30:30,933 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/5fb18f8bc21a4dc1b98c0af588fef229 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229 2023-10-02T12:30:30,942 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229, entries=2, sequenceid=28, filesize=5.4 K 2023-10-02T12:30:30,943 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 7ec576d712d0f3b0c0592584b88ce987 in 467ms, sequenceid=28, compaction requested=true 2023-10-02T12:30:30,943 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2520): Flush status journal for 7ec576d712d0f3b0c0592584b88ce987: 2023-10-02T12:30:30,943 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. for snapshot_backup_system completed. 2023-10-02T12:30:30,943 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:30:30,943 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:30:30,944 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255] hfiles 2023-10-02T12:30:30,944 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98 for snapshot=snapshot_backup_system 2023-10-02T12:30:30,944 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255 for snapshot=snapshot_backup_system 2023-10-02T12:30:30,944 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229] hfiles 2023-10-02T12:30:30,944 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0 for snapshot=snapshot_backup_system 2023-10-02T12:30:30,944 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c for snapshot=snapshot_backup_system 2023-10-02T12:30:30,944 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229 for snapshot=snapshot_backup_system 2023-10-02T12:30:30,952 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:30:30,952 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=61 2023-10-02T12:30:30,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=61 2023-10-02T12:30:30,952 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:30:30,955 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:30:30,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=61, resume processing ppid=60 2023-10-02T12:30:30,958 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=61, ppid=60, state=SUCCESS; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 in 637 msec 2023-10-02T12:30:30,958 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:30:30,959 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:30:30,960 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:30:30,960 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:30:30,961 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:30:30,978 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:30:30,988 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:30:30,989 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:30:30,990 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:30:30,990 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2023-10-02T12:30:30,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=60, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.0910 sec 2023-10-02T12:30:31,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=60 2023-10-02T12:30:31,010 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: backup:system, procId: 60 completed 2023-10-02T12:30:31,010 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(115): Backup backup_1696249829777 started at 1696249831010. 2023-10-02T12:30:31,012 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(119): Backup session backup_1696249829777 has been started. 2023-10-02T12:30:31,018 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:30:31,019 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:30:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:30:31,019 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:30:31,019 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:30:31,020 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:30:31,020 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:30:31,021 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:31,021 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:31,123 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:30:31,123 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:31,123 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:30:31,123 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:30:31,124 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:31,124 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:30:31,124 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:31,124 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:31,125 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:30:31,125 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:31,125 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:30:31,126 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:30:31,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:30:31,126 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:30:31,126 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:30:31,127 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:30:31,127 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:30:31,127 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:30:31,254 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:31,254 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:31,254 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:31,254 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:30:31,254 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:30:31,254 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-10-02T12:30:31,254 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:30:31,255 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:30:31,255 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:30:31,255 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:31,256 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:31,256 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:30:31,256 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:30:31,257 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@1a9b48a4[Count = 0] remaining members to acquire global barrier 2023-10-02T12:30:31,257 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:30:31,257 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:31,361 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:31,362 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:31,362 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:31,362 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:30:31,362 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:31,362 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:30:31,362 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:30:31,362 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:31,362 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696249819302 highest: 1696249819302 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:31,362 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114:(num 1696249819302) roll requested 2023-10-02T12:30:31,380 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249831362, exclude list is [], retry=0 2023-10-02T12:30:31,389 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:30:31,392 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249819302 with entries=17, filesize=5.25 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249831362 2023-10-02T12:30:31,392 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:30:31,392 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249819302 is not closed yet, will try archiving it next time 2023-10-02T12:30:31,392 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta:.meta(num 1696249819322) roll requested 2023-10-02T12:30:31,411 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249831392.meta, exclude list is [], retry=0 2023-10-02T12:30:31,416 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:30:31,422 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249819322.meta with entries=12, filesize=3.07 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249831392.meta 2023-10-02T12:30:31,422 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:30:31,422 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249819322.meta is not closed yet, will try archiving it next time 2023-10-02T12:30:31,423 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-10-02T12:30:31,423 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696249831362 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:31,441 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:30:31,441 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,38255,1696249782114' in zk 2023-10-02T12:30:31,578 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:31,578 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:30:31,578 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:30:31,578 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:30:31,580 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:31,580 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:30:31,580 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:30:31,580 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:30:31,581 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:30:31,581 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:31,582 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:31,582 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:30:31,583 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:31,583 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:31,584 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,38255,1696249782114': 2023-10-02T12:30:31,584 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:30:31,584 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:30:31,584 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:30:31,584 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:30:31,584 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:30:31,584 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:30:31,646 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:30:31,646 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:30:31,646 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:30:31,646 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:31,647 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:30:31,647 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:30:31,647 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:31,648 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:30:31,648 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:31,648 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:30:31,648 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:30:31,649 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:30:31,649 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:31,650 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:31,654 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:30:31,654 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:31,655 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:31,655 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:31,655 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:30:31,656 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:31,657 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:31,777 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:31,777 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-10-02T12:30:31,777 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:31,777 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): regionserver:38255-0x1017326ef4b0001, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-10-02T12:30:31,777 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:31,777 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:30:31,777 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:31,777 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:31,777 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:30:31,777 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:30:31,777 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:30:31,777 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:30:31,777 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(347): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Unable to get data of znode /1/rolllog-proc/abort/rolllog because node does not exist (not an error) 2023-10-02T12:30:31,778 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:31,778 DEBUG [Listener at localhost/45111-EventThread {}] zookeeper.ZKWatcher(604): master:40619-0x1017326ef4b0000, quorum=127.0.0.1:57768, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-10-02T12:30:31,779 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:30:31,779 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin(2838): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-10-02T12:30:31,779 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:30:31,779 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin(2846): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-10-02T12:30:31,879 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin(2852): Getting current status of procedure from master... 2023-10-02T12:30:31,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:30:31,884 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249831883_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } 2023-10-02T12:30:31,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249831884 (current time:1696249831884). 2023-10-02T12:30:31,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:30:31,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249831883_default_test-1696249791655 VERSION not specified, setting to 2 2023-10-02T12:30:31,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x1ca05907 to 127.0.0.1:57768 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-10-02T12:30:31,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f8cf2a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:31,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:31,997 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:40654, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x1ca05907 to 127.0.0.1:57768 2023-10-02T12:30:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:31,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:30:32,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=62, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249831883_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } 2023-10-02T12:30:32,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249831883_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-10-02T12:30:32,003 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249831883_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:30:32,004 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249831883_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:30:32,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-10-02T12:30:32,008 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249831883_default_test-1696249791655 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:30:32,011 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249831883_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:30:32,024 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249831883_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:30:32,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure c8f99eaa6d1b90df2e9c811158390413}] 2023-10-02T12:30:32,027 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:32,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-10-02T12:30:32,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=63 2023-10-02T12:30:32,178 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. 2023-10-02T12:30:32,179 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2520): Flush status journal for c8f99eaa6d1b90df2e9c811158390413: 2023-10-02T12:30:32,179 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. for snapshot_1696249831883_default_test-1696249791655 completed. 2023-10-02T12:30:32,179 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(240): Storing 'test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413.' region-info for snapshot=snapshot_1696249831883_default_test-1696249791655 2023-10-02T12:30:32,179 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:30:32,179 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_] hfiles 2023-10-02T12:30:32,179 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ for snapshot=snapshot_1696249831883_default_test-1696249791655 2023-10-02T12:30:32,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-10-02T12:30:32,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. 2023-10-02T12:30:32,586 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=63 2023-10-02T12:30:32,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=63 2023-10-02T12:30:32,587 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249831883_default_test-1696249791655 on region c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:32,588 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:32,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=63, resume processing ppid=62 2023-10-02T12:30:32,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=63, ppid=62, state=SUCCESS; SnapshotRegionProcedure c8f99eaa6d1b90df2e9c811158390413 in 565 msec 2023-10-02T12:30:32,591 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249831883_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:30:32,593 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249831883_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:30:32,594 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249831883_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:30:32,594 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249831883_default_test-1696249791655 2023-10-02T12:30:32,595 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249831883_default_test-1696249791655 2023-10-02T12:30:32,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-10-02T12:30:32,638 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249831883_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:30:32,649 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249831883_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:30:32,650 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249831883_default_test-1696249791655 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249831883_default_test-1696249791655 2023-10-02T12:30:32,652 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249831883_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:30:32,652 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249831883_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-10-02T12:30:32,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=62, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1696249831883_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } in 653 msec 2023-10-02T12:30:33,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=62 2023-10-02T12:30:33,111 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.HBaseAdmin$TableFuture(3677): Operation: SNAPSHOT, Table Name: default:test-1696249791655, procId: 62 completed 2023-10-02T12:30:33,111 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696249829777 2023-10-02T12:30:33,111 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:30:33,111 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-10-02T12:30:33,111 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249829777_test-1696249791655 2023-10-02T12:30:33,112 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249831883_default_test-1696249791655 to hdfs://localhost:37823/backupUT/backup_1696249829777/default/test-1696249791655/ 2023-10-02T12:30:33,112 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:30:33,139 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:30:33,139 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,40619,1696249778965_1499930026_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37823/backupUT/backup_1696249829777/default/test-1696249791655, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37823/backupUT/backup_1696249829777/default/test-1696249791655/.hbase-snapshot/.tmp/snapshot_1696249831883_default_test-1696249791655 2023-10-02T12:30:33,140 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07, snapshotDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249831883_default_test-1696249791655. 2023-10-02T12:30:33,149 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249831883_default_test-1696249791655 to hdfs://localhost:37823/backupUT/backup_1696249829777/default/test-1696249791655/.hbase-snapshot/.tmp/snapshot_1696249831883_default_test-1696249791655 2023-10-02T12:30:33,580 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:33,580 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:33,581 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:33,581 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:34,606 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-4750040518096259446.jar 2023-10-02T12:30:34,606 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:34,606 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:34,607 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:34,608 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:34,608 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:34,608 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:34,609 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:34,609 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:34,609 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:34,610 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:30:34,611 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:30:34,611 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:30:34,612 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:30:34,612 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:30:34,613 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:30:34,613 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:30:34,613 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:30:34,614 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:30:34,614 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:30:34,615 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:30:34,615 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:30:34,616 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:34,616 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:34,617 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:34,618 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:34,618 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:34,619 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:34,620 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:34,656 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-10-02T12:30:34,677 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249831883_default_test-1696249791655' hfile list 2023-10-02T12:30:34,683 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-10-02T12:30:34,794 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249834742/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:30:34,794 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249834742/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:30:34,870 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:30:34,895 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ output=hdfs://localhost:37823/backupUT/backup_1696249829777/default/test-1696249791655/archive/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ 2023-10-02T12:30:34,895 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 4.160M/sec 2023-10-02T12:30:35,853 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:30:35,854 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:30:35,868 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249831883_default_test-1696249791655 2023-10-02T12:30:35,869 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249831883_default_test-1696249791655 finished. 2023-10-02T12:30:35,869 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1034): test-1696249791655 2023-10-02T12:30:35,894 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249829777 2023-10-02T12:30:35,895 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:30:35,902 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37823/backupUT/backup_1696249829777/.backup.manifest 2023-10-02T12:30:35,902 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(358): Backup backup_1696249829777 finished: type=FULL,tablelist=test-1696249791655,targetRootDir=hdfs://localhost:37823/backupUT,startts=1696249831010,completets=1696249835894,bytescopied=0 2023-10-02T12:30:35,902 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:30:35,902 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249831883_default_test-1696249791655 2023-10-02T12:30:35,903 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249831883_default_test-1696249791655" 2023-10-02T12:30:35,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249831883_default_test-1696249791655 2023-10-02T12:30:35,908 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249831883_default_test-1696249791655 for backup backup_1696249829777 succeeded. 2023-10-02T12:30:35,909 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:30:35,918 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:30:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:30:35,923 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:30:35,926 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:30:35,935 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(376): Backup backup_1696249829777 completed. 2023-10-02T12:30:35,935 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] client.ConnectionImplementation(2120): Closing master protocol: MasterService 2023-10-02T12:30:35,935 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5fda4e0c to 127.0.0.1:57768 2023-10-02T12:30:35,935 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:35,951 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestFullRestore(285): backup complete You have to specify either set name or table list to restore Usage: hbase restore [options] backup_path Path to a backup destination root backup_id Backup image ID to restore table(s) Comma-separated list of tables to restore Options: -c Check restore sequence and dependencies only (does not execute the command) -d Enable debug loggings -h,--help Show usage -m A comma separated list of target tables. If specified, each table in must have a mapping -o Overwrite data if any of the restore target tables exists -q Yarn queue name to run backup restore command on -s Backup set to restore, mutually exclusive with -t (table list) -t Table name list, comma-separated. To enable backup, in hbase-site.xml, set: hbase.backup.enable=true hbase.master.logcleaner.plugins=YOUR_PLUGINS,org.apache.hadoop.hbase.backup.master.BackupLogCleaner hbase.procedure.master.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager hbase.procedure.regionserver.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager hbase.coprocessor.region.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.BackupObserver and restart the cluster For more information please see http://hbase.apache.org/book.html#backuprestore 2023-10-02T12:30:35,995 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleDNECommand Thread=599 (was 599), OpenFileDescriptor=691 (was 695), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=666 (was 672), ProcessCount=181 (was 181), AvailableMemoryMB=4176 (was 4369) 2023-10-02T12:30:35,995 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=599 is superior to 500 2023-10-02T12:30:36,018 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreCheckCommand Thread=599, OpenFileDescriptor=691, MaxFileDescriptor=60000, SystemLoadAverage=666, ProcessCount=181, AvailableMemoryMB=4175 2023-10-02T12:30:36,018 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=599 is superior to 500 2023-10-02T12:30:36,018 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestFullRestore(97): test full restore on a single table: command-line, check only 2023-10-02T12:30:36,026 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a81bfac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:36,042 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:36,044 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:40666, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:36,046 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:30:36,048 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59878, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:30:36,060 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:30:36,065 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:30:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249836065 (current time:1696249836065). 2023-10-02T12:30:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:30:36,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:30:36,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dfd3833, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:36,078 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:40682, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:30:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:30:36,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-10-02T12:30:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-10-02T12:30:36,083 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:30:36,086 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:30:36,090 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:30:36,094 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:30:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-10-02T12:30:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-10-02T12:30:36,502 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:30:36,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987}] 2023-10-02T12:30:36,504 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:30:36,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=65 2023-10-02T12:30:36,657 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:30:36,657 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2819): Flushing 7ec576d712d0f3b0c0592584b88ce987 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-10-02T12:30:36,662 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/b9b47a46600b4a0483abe90aca8688e8 is 169, key is trslm:hdfs://localhost:37823/backupUT\x00test-1696249791655/meta:log-roll-map/1696249835882/Put/seqid=0 2023-10-02T12:30:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-10-02T12:30:36,693 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/b9b47a46600b4a0483abe90aca8688e8 2023-10-02T12:30:36,705 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/d30d36b7102e4495bb9961fcaa5fc423 is 452, key is session:backup_1696249829777/session:context/1696249835924/Put/seqid=0 2023-10-02T12:30:36,740 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/d30d36b7102e4495bb9961fcaa5fc423 2023-10-02T12:30:36,751 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/b9b47a46600b4a0483abe90aca8688e8 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8 2023-10-02T12:30:36,771 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8, entries=4, sequenceid=39, filesize=5.5 K 2023-10-02T12:30:36,773 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/d30d36b7102e4495bb9961fcaa5fc423 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423 2023-10-02T12:30:36,783 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423, entries=2, sequenceid=39, filesize=5.4 K 2023-10-02T12:30:36,784 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 7ec576d712d0f3b0c0592584b88ce987 in 127ms, sequenceid=39, compaction requested=true 2023-10-02T12:30:36,784 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2520): Flush status journal for 7ec576d712d0f3b0c0592584b88ce987: 2023-10-02T12:30:36,784 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. for snapshot_backup_system completed. 2023-10-02T12:30:36,785 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:30:36,785 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:30:36,785 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8] hfiles 2023-10-02T12:30:36,785 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98 for snapshot=snapshot_backup_system 2023-10-02T12:30:36,785 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255 for snapshot=snapshot_backup_system 2023-10-02T12:30:36,785 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8 for snapshot=snapshot_backup_system 2023-10-02T12:30:36,785 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423] hfiles 2023-10-02T12:30:36,785 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0 for snapshot=snapshot_backup_system 2023-10-02T12:30:36,785 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c for snapshot=snapshot_backup_system 2023-10-02T12:30:36,785 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229 for snapshot=snapshot_backup_system 2023-10-02T12:30:36,785 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423 for snapshot=snapshot_backup_system 2023-10-02T12:30:36,839 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:30:36,840 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=65 2023-10-02T12:30:36,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=65 2023-10-02T12:30:36,840 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:30:36,842 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:30:36,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=65, resume processing ppid=64 2023-10-02T12:30:36,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=65, ppid=64, state=SUCCESS; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 in 341 msec 2023-10-02T12:30:36,845 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:30:36,846 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:30:36,847 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:30:36,847 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:30:36,848 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:30:36,871 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:30:36,886 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:30:36,891 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:30:36,893 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:30:36,893 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-10-02T12:30:36,895 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 813 msec 2023-10-02T12:30:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=64 2023-10-02T12:30:37,192 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(115): Backup backup_1696249836026 started at 1696249837192. 2023-10-02T12:30:37,194 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(119): Backup session backup_1696249836026 has been started. 2023-10-02T12:30:37,209 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:30:37,210 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:30:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-10-02T12:30:37,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:30:37,216 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:30:37,216 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:30:37,221 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:30:37,221 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:30:37,226 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:37,228 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:30:37,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:30:37,229 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:37,230 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:30:37,230 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:37,230 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:30:37,234 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:30:37,234 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-10-02T12:30:37,234 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:30:37,234 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:37,236 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:30:37,238 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:30:37,238 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:30:37,242 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:30:37,242 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:30:37,242 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:30:37,247 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:37,247 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:37,247 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:30:37,247 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:30:37,248 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:30:37,248 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:30:37,248 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:30:37,249 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:37,249 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:37,250 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:30:37,250 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:30:37,250 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@346be826[Count = 0] remaining members to acquire global barrier 2023-10-02T12:30:37,254 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:30:37,254 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:37,255 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:37,255 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:37,256 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:30:37,256 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:30:37,257 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:30:37,257 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:37,257 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696249831362 highest: 1696249831362 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:37,257 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114:(num 1696249831362) roll requested 2023-10-02T12:30:37,295 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249837258, exclude list is [], retry=0 2023-10-02T12:30:37,323 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:30:37,336 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249831362 with entries=10, filesize=2.31 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249837258 2023-10-02T12:30:37,336 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:30:37,336 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249831362 is not closed yet, will try archiving it next time 2023-10-02T12:30:37,336 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta:.meta(num 1696249831392) roll requested 2023-10-02T12:30:37,354 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249837336.meta, exclude list is [], retry=0 2023-10-02T12:30:37,363 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:30:37,368 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 111 2023-10-02T12:30:37,368 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696249837258 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:37,368 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249831392.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249837336.meta 2023-10-02T12:30:37,368 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:30:37,368 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249831392.meta is not closed yet, will try archiving it next time 2023-10-02T12:30:37,372 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249831392.meta not finished, retry = 0 2023-10-02T12:30:37,394 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:30:37,394 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,38255,1696249782114' in zk 2023-10-02T12:30:37,400 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:30:37,400 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:30:37,400 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:30:37,402 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:37,402 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:30:37,402 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:30:37,405 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:30:37,410 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:30:37,410 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:37,411 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:37,412 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:30:37,412 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:37,414 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:37,416 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,38255,1696249782114': 2023-10-02T12:30:37,416 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:30:37,416 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:30:37,416 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:30:37,416 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:30:37,416 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:30:37,416 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:30:37,418 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:30:37,418 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:37,418 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:37,418 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:30:37,418 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:30:37,419 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:30:37,419 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:30:37,420 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:30:37,420 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:37,421 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:37,421 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:30:37,422 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:37,426 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:37,429 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:30:37,429 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:30:37,429 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:30:37,429 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:30:37,429 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:30:37,429 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:30:37,430 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:30:37,430 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:30:37,474 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249831392.meta to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/oldWALs/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249831392.meta 2023-10-02T12:30:37,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:30:37,534 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249837533_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } 2023-10-02T12:30:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249837534 (current time:1696249837534). 2023-10-02T12:30:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:30:37,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249837533_default_test-1696249791655 VERSION not specified, setting to 2 2023-10-02T12:30:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bd3f008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:37,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:37,544 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:40684, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:37,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:37,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:30:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249837533_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } 2023-10-02T12:30:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249837533_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-10-02T12:30:37,552 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249837533_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:30:37,553 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249837533_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:30:37,570 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249837533_default_test-1696249791655 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:30:37,574 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249837533_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:30:37,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-10-02T12:30:37,583 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249837533_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:30:37,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure c8f99eaa6d1b90df2e9c811158390413}] 2023-10-02T12:30:37,586 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:37,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-10-02T12:30:37,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=67 2023-10-02T12:30:37,737 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. 2023-10-02T12:30:37,738 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(2520): Flush status journal for c8f99eaa6d1b90df2e9c811158390413: 2023-10-02T12:30:37,738 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. for snapshot_1696249837533_default_test-1696249791655 completed. 2023-10-02T12:30:37,738 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(240): Storing 'test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413.' region-info for snapshot=snapshot_1696249837533_default_test-1696249791655 2023-10-02T12:30:37,738 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:30:37,738 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_] hfiles 2023-10-02T12:30:37,738 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ for snapshot=snapshot_1696249837533_default_test-1696249791655 2023-10-02T12:30:37,745 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. 2023-10-02T12:30:37,745 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=67 2023-10-02T12:30:37,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=67 2023-10-02T12:30:37,746 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249837533_default_test-1696249791655 on region c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:37,747 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:37,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=67, resume processing ppid=66 2023-10-02T12:30:37,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=67, ppid=66, state=SUCCESS; SnapshotRegionProcedure c8f99eaa6d1b90df2e9c811158390413 in 165 msec 2023-10-02T12:30:37,750 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249837533_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:30:37,751 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249837533_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:30:37,752 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249837533_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:30:37,752 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249837533_default_test-1696249791655 2023-10-02T12:30:37,753 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249837533_default_test-1696249791655 2023-10-02T12:30:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-10-02T12:30:38,172 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249837533_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:30:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-10-02T12:30:38,179 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249837533_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:30:38,180 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249837533_default_test-1696249791655 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249837533_default_test-1696249791655 2023-10-02T12:30:38,181 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249837533_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:30:38,181 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249837533_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-10-02T12:30:38,182 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=66, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1696249837533_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } in 635 msec 2023-10-02T12:30:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=66 2023-10-02T12:30:38,680 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696249836026 2023-10-02T12:30:38,681 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:30:38,681 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-10-02T12:30:38,681 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249836026_test-1696249791655 2023-10-02T12:30:38,681 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249837533_default_test-1696249791655 to hdfs://localhost:37823/backupUT/backup_1696249836026/default/test-1696249791655/ 2023-10-02T12:30:38,681 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:30:38,704 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:30:38,704 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,40619,1696249778965_943524805_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37823/backupUT/backup_1696249836026/default/test-1696249791655, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37823/backupUT/backup_1696249836026/default/test-1696249791655/.hbase-snapshot/.tmp/snapshot_1696249837533_default_test-1696249791655 2023-10-02T12:30:38,704 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07, snapshotDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249837533_default_test-1696249791655. 2023-10-02T12:30:38,713 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249837533_default_test-1696249791655 to hdfs://localhost:37823/backupUT/backup_1696249836026/default/test-1696249791655/.hbase-snapshot/.tmp/snapshot_1696249837533_default_test-1696249791655 2023-10-02T12:30:39,131 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:39,131 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:39,132 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:39,132 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:40,328 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-799409477703664900.jar 2023-10-02T12:30:40,329 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:40,329 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:40,330 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:40,330 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:40,331 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:40,331 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:40,332 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:40,332 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:40,333 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:40,334 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:30:40,335 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:30:40,336 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:30:40,336 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:30:40,337 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:30:40,338 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:30:40,338 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:30:40,339 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:30:40,340 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:30:40,340 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:30:40,341 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:30:40,342 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:30:40,343 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:40,343 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:40,344 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:40,345 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:40,345 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:40,346 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:40,347 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:40,415 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249837533_default_test-1696249791655' hfile list 2023-10-02T12:30:40,418 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-10-02T12:30:40,534 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249840483/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:30:40,535 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249840483/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:30:40,623 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:30:40,703 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ output=hdfs://localhost:37823/backupUT/backup_1696249836026/default/test-1696249791655/archive/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ 2023-10-02T12:30:40,703 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 4.160M/sec 2023-10-02T12:30:41,589 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:30:41,590 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:30:41,598 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249837533_default_test-1696249791655 2023-10-02T12:30:41,599 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249837533_default_test-1696249791655 finished. 2023-10-02T12:30:41,599 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1034): test-1696249791655 2023-10-02T12:30:41,608 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249836026 2023-10-02T12:30:41,608 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:30:42,023 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37823/backupUT/backup_1696249836026/.backup.manifest 2023-10-02T12:30:42,023 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(358): Backup backup_1696249836026 finished: type=FULL,tablelist=test-1696249791655,targetRootDir=hdfs://localhost:37823/backupUT,startts=1696249837192,completets=1696249841608,bytescopied=0 2023-10-02T12:30:42,023 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:30:42,023 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249837533_default_test-1696249791655 2023-10-02T12:30:42,025 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249837533_default_test-1696249791655" 2023-10-02T12:30:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249837533_default_test-1696249791655 2023-10-02T12:30:42,029 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249837533_default_test-1696249791655 for backup backup_1696249836026 succeeded. 2023-10-02T12:30:42,030 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:30:42,036 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:30:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:30:42,041 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:30:42,043 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:30:42,051 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(376): Backup backup_1696249836026 completed. 2023-10-02T12:30:42,052 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:42,052 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestFullRestore(101): backup complete 2023-10-02T12:30:42,067 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] backup.RestoreDriver(108): Found -check option in restore command, will check and verify the dependencies 2023-10-02T12:30:42,077 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54931bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:42,080 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249836026/.backup.manifest 2023-10-02T12:30:42,083 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249836026/.backup.manifest 2023-10-02T12:30:42,083 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] util.BackupUtils(677): Dependent image(s) from old to new: 2023-10-02T12:30:42,084 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] util.BackupUtils(686): Backup image: backup_1696249836026 for 'test-1696249791655' is available 2023-10-02T12:30:42,084 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupAdminImpl(511): Checking backup images: OK 2023-10-02T12:30:42,084 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:42,111 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreCheckCommand Thread=600 (was 599) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57768@0x48eb4f1e java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$482/273044086.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x3eb3393a-shared-pool-12 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57768@0x7ed409f4-SendThread(127.0.0.1:57768) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2000333002_16 at /127.0.0.1:55414 [Receiving block BP-2025135827-172.31.12.81-1696249775150:blk_1073741916_1092] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x3eb3393a-shared-pool-13 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1482744570_16 at /127.0.0.1:55386 [Waiting for operation #18] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-2025135827-172.31.12.81-1696249775150:blk_1073741916_1092, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_2000333002_16 at /127.0.0.1:55412 [Receiving block BP-2025135827-172.31.12.81-1696249775150:blk_1073741915_1091] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: WAL-Archive-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57768@0x7ed409f4 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1455) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1485) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1415) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$482/273044086.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050044864_16 at /127.0.0.1:55452 [Waiting for operation #8] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x712cc456-shared-pool-29 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-2025135827-172.31.12.81-1696249775150:blk_1073741915_1091, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57768@0x7ed409f4-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57768@0x48eb4f1e-SendThread(127.0.0.1:57768) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:228) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1338) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1261) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57768@0x48eb4f1e-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) - Thread LEAK? -, OpenFileDescriptor=696 (was 691) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=648 (was 666), ProcessCount=181 (was 181), AvailableMemoryMB=3630 (was 4175) 2023-10-02T12:30:42,112 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=600 is superior to 500 2023-10-02T12:30:42,136 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingle Thread=600, OpenFileDescriptor=696, MaxFileDescriptor=60000, SystemLoadAverage=648, ProcessCount=181, AvailableMemoryMB=3630 2023-10-02T12:30:42,136 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=600 is superior to 500 2023-10-02T12:30:42,136 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestFullRestore(55): test full restore on a single table empty table 2023-10-02T12:30:42,148 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b0fa2bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:42,155 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:42,182 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42804, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:42,210 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:30:42,215 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48624, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:30:42,255 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:30:42,264 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:30:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249842264 (current time:1696249842264). 2023-10-02T12:30:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:30:42,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:30:42,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@218aa92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:42,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:42,278 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42810, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:42,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:42,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:30:42,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=68, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:30:42,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 68 2023-10-02T12:30:42,285 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:30:42,286 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:30:42,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-10-02T12:30:42,291 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:30:42,294 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:30:42,311 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:30:42,311 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987}] 2023-10-02T12:30:42,312 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=69, ppid=68, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:30:42,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-10-02T12:30:42,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=69 2023-10-02T12:30:42,465 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:30:42,465 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegion(2819): Flushing 7ec576d712d0f3b0c0592584b88ce987 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-10-02T12:30:42,471 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/85d2568c07cc4be686b8e706b89254ce is 169, key is trslm:hdfs://localhost:37823/backupUT\x00test-1696249791655/meta:log-roll-map/1696249841602/Put/seqid=0 2023-10-02T12:30:42,480 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/85d2568c07cc4be686b8e706b89254ce 2023-10-02T12:30:42,494 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/f8db732eb5cd417eaa236f25fd437245 is 452, key is session:backup_1696249836026/session:context/1696249842042/Put/seqid=0 2023-10-02T12:30:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-10-02T12:30:42,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-10-02T12:30:42,925 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/f8db732eb5cd417eaa236f25fd437245 2023-10-02T12:30:42,936 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/85d2568c07cc4be686b8e706b89254ce as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/85d2568c07cc4be686b8e706b89254ce 2023-10-02T12:30:42,943 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/85d2568c07cc4be686b8e706b89254ce, entries=4, sequenceid=50, filesize=5.5 K 2023-10-02T12:30:42,944 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/f8db732eb5cd417eaa236f25fd437245 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/f8db732eb5cd417eaa236f25fd437245 2023-10-02T12:30:42,951 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/f8db732eb5cd417eaa236f25fd437245, entries=2, sequenceid=50, filesize=5.4 K 2023-10-02T12:30:42,952 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 7ec576d712d0f3b0c0592584b88ce987 in 487ms, sequenceid=50, compaction requested=true 2023-10-02T12:30:42,952 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegion(2520): Flush status journal for 7ec576d712d0f3b0c0592584b88ce987: 2023-10-02T12:30:42,952 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. for snapshot_backup_system completed. 2023-10-02T12:30:42,953 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:30:42,953 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:30:42,953 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/85d2568c07cc4be686b8e706b89254ce] hfiles 2023-10-02T12:30:42,953 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98 for snapshot=snapshot_backup_system 2023-10-02T12:30:42,953 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255 for snapshot=snapshot_backup_system 2023-10-02T12:30:42,953 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8 for snapshot=snapshot_backup_system 2023-10-02T12:30:42,953 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/85d2568c07cc4be686b8e706b89254ce for snapshot=snapshot_backup_system 2023-10-02T12:30:42,953 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/f8db732eb5cd417eaa236f25fd437245] hfiles 2023-10-02T12:30:42,953 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0 for snapshot=snapshot_backup_system 2023-10-02T12:30:42,953 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c for snapshot=snapshot_backup_system 2023-10-02T12:30:42,953 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229 for snapshot=snapshot_backup_system 2023-10-02T12:30:42,953 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423 for snapshot=snapshot_backup_system 2023-10-02T12:30:42,953 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/f8db732eb5cd417eaa236f25fd437245 for snapshot=snapshot_backup_system 2023-10-02T12:30:43,360 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:30:43,360 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=69 2023-10-02T12:30:43,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=69 2023-10-02T12:30:43,361 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:30:43,364 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=69, ppid=68, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:30:43,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=69, resume processing ppid=68 2023-10-02T12:30:43,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=69, ppid=68, state=SUCCESS; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 in 1.0540 sec 2023-10-02T12:30:43,367 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:30:43,368 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:30:43,368 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:30:43,368 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:30:43,369 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:30:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-10-02T12:30:43,395 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:30:43,413 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:30:43,413 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:30:43,416 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:30:43,417 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 68 2023-10-02T12:30:43,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=68, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.1350 sec 2023-10-02T12:30:44,184 DEBUG [regionserver/jenkins-hbase3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.15384615384615385, tune throughput to 57.69 MB/second 2023-10-02T12:30:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=68 2023-10-02T12:30:44,397 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(115): Backup backup_1696249842148 started at 1696249844396. 2023-10-02T12:30:44,398 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(119): Backup session backup_1696249842148 has been started. 2023-10-02T12:30:44,404 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:30:44,405 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:30:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:30:44,405 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:30:44,405 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:30:44,406 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:30:44,406 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:30:44,407 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:44,410 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:44,410 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:30:44,410 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:30:44,414 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:30:44,414 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:44,415 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:30:44,415 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:44,415 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:30:44,415 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:30:44,415 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:30:44,415 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:30:44,415 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:30:44,416 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:30:44,416 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:30:44,416 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:30:44,418 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:44,419 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:44,419 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:30:44,419 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:30:44,422 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:30:44,422 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:30:44,429 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:30:44,429 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:44,432 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:44,432 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:30:44,437 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:30:44,437 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@3359955[Count = 0] remaining members to acquire global barrier 2023-10-02T12:30:44,437 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:30:44,437 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:44,438 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:44,439 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:44,439 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:30:44,439 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:30:44,439 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:30:44,439 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:44,439 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696249837258 highest: 1696249837258 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:44,439 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114:(num 1696249837258) roll requested 2023-10-02T12:30:44,451 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249844439, exclude list is [], retry=0 2023-10-02T12:30:44,455 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:30:44,457 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249837258 with entries=10, filesize=2.31 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249844439 2023-10-02T12:30:44,457 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:30:44,457 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249837258 is not closed yet, will try archiving it next time 2023-10-02T12:30:44,458 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta:.meta(num 1696249837336) roll requested 2023-10-02T12:30:44,458 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249831362 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/oldWALs/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249831362 2023-10-02T12:30:44,471 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249844458.meta, exclude list is [], retry=0 2023-10-02T12:30:44,474 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:30:44,476 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249837336.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249844458.meta 2023-10-02T12:30:44,476 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:30:44,477 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249837336.meta is not closed yet, will try archiving it next time 2023-10-02T12:30:44,479 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-10-02T12:30:44,480 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696249844439 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:44,480 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249837336.meta to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/oldWALs/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249837336.meta 2023-10-02T12:30:44,491 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:30:44,491 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,38255,1696249782114' in zk 2023-10-02T12:30:44,494 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:44,494 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:30:44,494 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:30:44,494 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:30:44,494 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:30:44,495 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:30:44,496 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:30:44,497 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:30:44,497 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:44,497 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:44,502 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:30:44,502 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:44,503 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:44,504 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,38255,1696249782114': 2023-10-02T12:30:44,504 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:30:44,504 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:30:44,504 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:30:44,504 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:30:44,504 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:30:44,504 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:30:44,508 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:44,508 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:30:44,508 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:30:44,508 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:30:44,508 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:44,508 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:30:44,508 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:30:44,509 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:44,509 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:30:44,509 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:44,511 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:30:44,518 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:44,522 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:44,530 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:30:44,530 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:30:44,531 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:30:44,531 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:30:44,532 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:30:44,532 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:30:44,532 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:30:44,533 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:30:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:30:44,638 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249844637_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } 2023-10-02T12:30:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249844638 (current time:1696249844638). 2023-10-02T12:30:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:30:44,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249844637_default_test-1696249791655 VERSION not specified, setting to 2 2023-10-02T12:30:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7704274, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:44,657 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:42818, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:30:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=70, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1696249844637_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } 2023-10-02T12:30:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249844637_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 }, snapshot procedure id = 70 2023-10-02T12:30:44,663 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1696249844637_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:30:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=70 2023-10-02T12:30:44,665 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1696249844637_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:30:44,668 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249844637_default_test-1696249791655 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:30:44,670 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1696249844637_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:30:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=70 2023-10-02T12:30:44,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=70 2023-10-02T12:30:45,077 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1696249844637_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:30:45,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE; SnapshotRegionProcedure c8f99eaa6d1b90df2e9c811158390413}] 2023-10-02T12:30:45,080 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=71, ppid=70, state=RUNNABLE; SnapshotRegionProcedure c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:45,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=71 2023-10-02T12:30:45,232 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. 2023-10-02T12:30:45,232 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.HRegion(2520): Flush status journal for c8f99eaa6d1b90df2e9c811158390413: 2023-10-02T12:30:45,232 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. for snapshot_1696249844637_default_test-1696249791655 completed. 2023-10-02T12:30:45,232 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(240): Storing 'test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413.' region-info for snapshot=snapshot_1696249844637_default_test-1696249791655 2023-10-02T12:30:45,232 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:30:45,232 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_] hfiles 2023-10-02T12:30:45,233 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ for snapshot=snapshot_1696249844637_default_test-1696249791655 2023-10-02T12:30:45,240 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. 2023-10-02T12:30:45,240 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=71 2023-10-02T12:30:45,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=71 2023-10-02T12:30:45,240 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249844637_default_test-1696249791655 on region c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:45,242 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=71, ppid=70, state=RUNNABLE; SnapshotRegionProcedure c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:45,245 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=71, resume processing ppid=70 2023-10-02T12:30:45,245 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1696249844637_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:30:45,245 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=71, ppid=70, state=SUCCESS; SnapshotRegionProcedure c8f99eaa6d1b90df2e9c811158390413 in 166 msec 2023-10-02T12:30:45,246 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1696249844637_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:30:45,246 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1696249844637_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:30:45,246 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249844637_default_test-1696249791655 2023-10-02T12:30:45,247 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249844637_default_test-1696249791655 2023-10-02T12:30:45,259 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1696249844637_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:30:45,268 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1696249844637_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:30:45,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=70 2023-10-02T12:30:45,271 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249844637_default_test-1696249791655 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249844637_default_test-1696249791655 2023-10-02T12:30:45,273 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1696249844637_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:30:45,273 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249844637_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 }, snapshot procedure id = 70 2023-10-02T12:30:45,274 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=70, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1696249844637_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } in 614 msec 2023-10-02T12:30:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=70 2023-10-02T12:30:45,771 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696249842148 2023-10-02T12:30:45,771 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:30:45,771 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-10-02T12:30:45,771 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249842148_test-1696249791655 2023-10-02T12:30:45,771 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249844637_default_test-1696249791655 to hdfs://localhost:37823/backupUT/backup_1696249842148/default/test-1696249791655/ 2023-10-02T12:30:45,771 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:30:45,796 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:30:45,796 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,40619,1696249778965_-22937581_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37823/backupUT/backup_1696249842148/default/test-1696249791655, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37823/backupUT/backup_1696249842148/default/test-1696249791655/.hbase-snapshot/.tmp/snapshot_1696249844637_default_test-1696249791655 2023-10-02T12:30:45,797 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07, snapshotDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249844637_default_test-1696249791655. 2023-10-02T12:30:45,809 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249844637_default_test-1696249791655 to hdfs://localhost:37823/backupUT/backup_1696249842148/default/test-1696249791655/.hbase-snapshot/.tmp/snapshot_1696249844637_default_test-1696249791655 2023-10-02T12:30:45,851 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:45,852 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:45,853 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:45,853 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:45,898 DEBUG [master/jenkins-hbase3:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region d30d346a1595b3fd84f804f2773895cc changed from -1.0 to 0.0, refreshing cache 2023-10-02T12:30:47,132 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-187570467651653444.jar 2023-10-02T12:30:47,133 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:47,134 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:47,134 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:47,135 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:47,136 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:47,136 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:47,137 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:47,137 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:47,138 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:47,139 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:30:47,140 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:30:47,140 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:30:47,141 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:30:47,142 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:30:47,142 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:30:47,144 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:30:47,145 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:30:47,146 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:30:47,146 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:30:47,147 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:30:47,148 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:30:47,148 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:47,149 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:47,150 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:47,151 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:47,151 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:47,159 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:47,161 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:47,242 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249844637_default_test-1696249791655' hfile list 2023-10-02T12:30:47,248 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-10-02T12:30:47,444 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249847367/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:30:47,444 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249847367/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:30:47,513 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:30:47,548 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ output=hdfs://localhost:37823/backupUT/backup_1696249842148/default/test-1696249791655/archive/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ 2023-10-02T12:30:47,548 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 4.160M/sec 2023-10-02T12:30:48,480 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:30:48,481 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:30:48,489 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249844637_default_test-1696249791655 2023-10-02T12:30:48,489 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249844637_default_test-1696249791655 finished. 2023-10-02T12:30:48,489 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1034): test-1696249791655 2023-10-02T12:30:48,501 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249842148 2023-10-02T12:30:48,502 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:30:48,910 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37823/backupUT/backup_1696249842148/.backup.manifest 2023-10-02T12:30:48,910 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(358): Backup backup_1696249842148 finished: type=FULL,tablelist=test-1696249791655,targetRootDir=hdfs://localhost:37823/backupUT,startts=1696249844396,completets=1696249848501,bytescopied=0 2023-10-02T12:30:48,910 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:30:48,910 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249844637_default_test-1696249791655 2023-10-02T12:30:48,911 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249844637_default_test-1696249791655" 2023-10-02T12:30:48,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249844637_default_test-1696249791655 2023-10-02T12:30:48,918 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249844637_default_test-1696249791655 for backup backup_1696249842148 succeeded. 2023-10-02T12:30:48,921 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:30:48,930 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:30:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:30:48,934 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:30:48,940 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:30:48,948 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(376): Backup backup_1696249842148 completed. 2023-10-02T12:30:48,949 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:48,998 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestFullRestore(61): backup complete 2023-10-02T12:30:49,002 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-10-02T12:30:49,003 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249842148/.backup.manifest 2023-10-02T12:30:49,006 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249842148/.backup.manifest 2023-10-02T12:30:49,006 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249842148/.backup.manifest 2023-10-02T12:30:49,009 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249842148/.backup.manifest 2023-10-02T12:30:49,009 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(148): Restoring 'test-1696249791655' to 'table1' from full backup image hdfs://localhost:37823/backupUT/backup_1696249842148/default/test-1696249791655 2023-10-02T12:30:49,024 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(487): Creating target table 'table1' 2023-10-02T12:30:49,025 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:37823/backupUT/backup_1696249842148/default/test-1696249791655/archive/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:49,026 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:37823/backupUT/backup_1696249842148/default/test-1696249791655/archive/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f in region [hdfs://localhost:37823/backupUT/backup_1696249842148/default/test-1696249791655/archive/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413] 2023-10-02T12:30:49,031 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:37823/backupUT/backup_1696249842148/default/test-1696249791655/archive/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ first=row0 last=row98 2023-10-02T12:30:49,032 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:30:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-10-02T12:30:49,036 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:30:49,036 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 72 2023-10-02T12:30:49,036 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:30:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-10-02T12:30:49,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-10-02T12:30:49,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-10-02T12:30:49,445 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => eadca8e9e0b0e864a1bf0e4e75a7b34c, NAME => 'table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:30:49,455 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:30:49,455 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing eadca8e9e0b0e864a1bf0e4e75a7b34c, disabling compactions & flushes 2023-10-02T12:30:49,455 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c. 2023-10-02T12:30:49,456 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c. 2023-10-02T12:30:49,456 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c. after waiting 0 ms 2023-10-02T12:30:49,456 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c. 2023-10-02T12:30:49,456 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c. 2023-10-02T12:30:49,456 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for eadca8e9e0b0e864a1bf0e4e75a7b34c: 2023-10-02T12:30:49,457 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:30:49,457 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1696249849457"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249849457"}]},"ts":"1696249849457"} 2023-10-02T12:30:49,459 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:30:49,460 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:30:49,460 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249849460"}]},"ts":"1696249849460"} 2023-10-02T12:30:49,461 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-10-02T12:30:49,466 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=eadca8e9e0b0e864a1bf0e4e75a7b34c, ASSIGN}] 2023-10-02T12:30:49,468 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=eadca8e9e0b0e864a1bf0e4e75a7b34c, ASSIGN 2023-10-02T12:30:49,469 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=eadca8e9e0b0e864a1bf0e4e75a7b34c, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38255,1696249782114; forceNewPlan=false, retain=false 2023-10-02T12:30:49,621 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=eadca8e9e0b0e864a1bf0e4e75a7b34c, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:49,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure eadca8e9e0b0e864a1bf0e4e75a7b34c, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:30:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-10-02T12:30:49,780 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(130): Open table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c. 2023-10-02T12:30:49,781 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7254): Opening region: {ENCODED => eadca8e9e0b0e864a1bf0e4e75a7b34c, NAME => 'table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:30:49,781 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:30:49,781 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 eadca8e9e0b0e864a1bf0e4e75a7b34c 2023-10-02T12:30:49,781 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(888): Instantiated table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:30:49,781 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7296): checking encryption for eadca8e9e0b0e864a1bf0e4e75a7b34c 2023-10-02T12:30:49,782 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7299): checking classloading for eadca8e9e0b0e864a1bf0e4e75a7b34c 2023-10-02T12:30:49,783 INFO [StoreOpener-eadca8e9e0b0e864a1bf0e4e75a7b34c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region eadca8e9e0b0e864a1bf0e4e75a7b34c 2023-10-02T12:30:49,785 INFO [StoreOpener-eadca8e9e0b0e864a1bf0e4e75a7b34c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eadca8e9e0b0e864a1bf0e4e75a7b34c columnFamilyName f 2023-10-02T12:30:49,786 DEBUG [StoreOpener-eadca8e9e0b0e864a1bf0e4e75a7b34c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:30:49,786 INFO [StoreOpener-eadca8e9e0b0e864a1bf0e4e75a7b34c-1 {}] regionserver.HStore(324): Store=eadca8e9e0b0e864a1bf0e4e75a7b34c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:30:49,787 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/eadca8e9e0b0e864a1bf0e4e75a7b34c 2023-10-02T12:30:49,788 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/eadca8e9e0b0e864a1bf0e4e75a7b34c 2023-10-02T12:30:49,792 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1076): writing seq id for eadca8e9e0b0e864a1bf0e4e75a7b34c 2023-10-02T12:30:49,795 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/eadca8e9e0b0e864a1bf0e4e75a7b34c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:30:49,796 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): Opened eadca8e9e0b0e864a1bf0e4e75a7b34c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11232220960, jitterRate=0.04608209431171417}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:30:49,797 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(992): Region open journal for eadca8e9e0b0e864a1bf0e4e75a7b34c: 2023-10-02T12:30:49,798 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2556): Post open deploy tasks for table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c., pid=74, masterSystemTime=1696249849775 2023-10-02T12:30:49,800 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2583): Finished post open deploy task for table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c. 2023-10-02T12:30:49,800 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(158): Opened table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c. 2023-10-02T12:30:49,801 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=eadca8e9e0b0e864a1bf0e4e75a7b34c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:49,804 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=74, resume processing ppid=73 2023-10-02T12:30:49,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure eadca8e9e0b0e864a1bf0e4e75a7b34c, server=jenkins-hbase3.apache.org,38255,1696249782114 in 180 msec 2023-10-02T12:30:49,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=73, resume processing ppid=72 2023-10-02T12:30:49,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=table1, region=eadca8e9e0b0e864a1bf0e4e75a7b34c, ASSIGN in 338 msec 2023-10-02T12:30:49,807 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:30:49,807 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249849807"}]},"ts":"1696249849807"} 2023-10-02T12:30:49,809 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-10-02T12:30:49,812 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:30:49,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=72, state=SUCCESS; CreateTableProcedure table=table1 in 780 msec 2023-10-02T12:30:50,135 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-10-02T12:30:50,135 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_table1 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:30:50,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=72 2023-10-02T12:30:50,175 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:37823/backupUT/backup_1696249842148/default/test-1696249791655/archive/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413 from hbase tables test-1696249791655 to tables table1 2023-10-02T12:30:50,175 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1696249791655 into table1 2023-10-02T12:30:50,177 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-table1-1696249850175 from hdfs://localhost:37823/backupUT/backup_1696249842148/default/test-1696249791655/archive/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:50,188 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46744e0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:50,189 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:30:50,190 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36818, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:30:50,191 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-10-02T12:30:50,191 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-10-02T12:30:50,201 DEBUG [hconnection-0x4f9abcb-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:50,202 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43278, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:50,204 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-10-02T12:30:50,204 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-10-02T12:30:50,205 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/partitions_0230ce3a-68a4-4f35-97ae-9e7ca3b4ff04 2023-10-02T12:30:50,217 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:50,218 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:50,219 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:50,219 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:50,600 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2023-10-02T12:30:51,270 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-1568116587879168436.jar 2023-10-02T12:30:51,271 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:51,271 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:51,271 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:51,272 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:51,272 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:51,272 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:51,273 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:51,273 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:51,273 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:51,274 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:30:51,275 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:30:51,275 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:30:51,275 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:30:51,276 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:30:51,276 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:30:51,277 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:30:51,277 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:30:51,278 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:30:51,278 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:30:51,279 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:30:51,279 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:30:51,279 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:51,280 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:51,280 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:51,280 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:51,281 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:51,281 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:51,282 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:51,282 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-10-02T12:30:51,282 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:51,282 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-10-02T12:30:51,282 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-10-02T12:30:51,283 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:30:51,296 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-10-02T12:30:51,477 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249851355/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:30:51,478 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249851355/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:30:51,623 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:37823/backupUT/backup_1696249842148/default/test-1696249791655/archive/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ 2023-10-02T12:30:51,629 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-10-02T12:30:51,700 DEBUG [pool-299-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51608063, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:51,702 DEBUG [hconnection-0x1d69df2b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:51,703 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43282, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:51,705 DEBUG [pool-299-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:51,707 DEBUG [pool-299-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-10-02T12:30:51,707 DEBUG [pool-299-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-10-02T12:30:51,718 DEBUG [pool-299-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-table1-1696249850175/_temporary/0/_temporary/attempt_local1267912811_0009_r_000000_0/f/0b5f54a5c8ee4a50818dafc27ca5fef0 is 37, key is row10/f:q1/1696249794327/Put/seqid=0 2023-10-02T12:30:51,720 WARN [Thread-1256 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:38255] Specified favored nodes: [jenkins-hbase3.apache.org:38255] 2023-10-02T12:30:52,546 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-table1-1696249850175 2023-10-02T12:30:52,552 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45980673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:52,554 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:52,555 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43288, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:52,563 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-table1-1696249850175/_SUCCESS 2023-10-02T12:30:52,567 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:30:52,568 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36830, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:30:52,578 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-table1-1696249850175/f/0b5f54a5c8ee4a50818dafc27ca5fef0 first=Optional[row0] last=Optional[row98] 2023-10-02T12:30:52,580 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c., hostname=jenkins-hbase3.apache.org,38255,1696249782114, seqNum=2 for row with hfile group [{f,hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-table1-1696249850175/f/0b5f54a5c8ee4a50818dafc27ca5fef0}] 2023-10-02T12:30:52,585 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-table1-1696249850175/f/0b5f54a5c8ee4a50818dafc27ca5fef0 for inclusion in eadca8e9e0b0e864a1bf0e4e75a7b34c/f 2023-10-02T12:30:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-10-02T12:30:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(625): Region bounds: first= last= 2023-10-02T12:30:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HRegion(2520): Flush status journal for eadca8e9e0b0e864a1bf0e4e75a7b34c: 2023-10-02T12:30:52,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-table1-1696249850175/f/0b5f54a5c8ee4a50818dafc27ca5fef0 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__table1__guq5jc2tci8cmh5ir51jt0pi3uk134g75ua8g5q0edl6jaj1anddm3e8prljsol1/f/0b5f54a5c8ee4a50818dafc27ca5fef0 2023-10-02T12:30:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__table1__guq5jc2tci8cmh5ir51jt0pi3uk134g75ua8g5q0edl6jaj1anddm3e8prljsol1/f/0b5f54a5c8ee4a50818dafc27ca5fef0 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/eadca8e9e0b0e864a1bf0e4e75a7b34c/f/e1d13a33e2c54e2db1e896f298b76aec_SeqId_4_ 2023-10-02T12:30:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44590505, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:30:52,604 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36836, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:30:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:52,609 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43300, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:52,625 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__table1__guq5jc2tci8cmh5ir51jt0pi3uk134g75ua8g5q0edl6jaj1anddm3e8prljsol1/f/0b5f54a5c8ee4a50818dafc27ca5fef0 into eadca8e9e0b0e864a1bf0e4e75a7b34c/f as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/eadca8e9e0b0e864a1bf0e4e75a7b34c/f/e1d13a33e2c54e2db1e896f298b76aec_SeqId_4_ - updating store file list. 2023-10-02T12:30:52,632 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/eadca8e9e0b0e864a1bf0e4e75a7b34c/f/e1d13a33e2c54e2db1e896f298b76aec_SeqId_4_ into eadca8e9e0b0e864a1bf0e4e75a7b34c/f 2023-10-02T12:30:52,632 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__table1__guq5jc2tci8cmh5ir51jt0pi3uk134g75ua8g5q0edl6jaj1anddm3e8prljsol1/f/0b5f54a5c8ee4a50818dafc27ca5fef0 into eadca8e9e0b0e864a1bf0e4e75a7b34c/f (new location: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/eadca8e9e0b0e864a1bf0e4e75a7b34c/f/e1d13a33e2c54e2db1e896f298b76aec_SeqId_4_) 2023-10-02T12:30:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__table1__guq5jc2tci8cmh5ir51jt0pi3uk134g75ua8g5q0edl6jaj1anddm3e8prljsol1/f/0b5f54a5c8ee4a50818dafc27ca5fef0 2023-10-02T12:30:52,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@678b89cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:30:52,644 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36850, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-10-02T12:30:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:52,650 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43308, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-10-02T12:30:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:52,683 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:52,683 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-10-02T12:30:52,683 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-10-02T12:30:52,683 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(233): Backup: backup_1696249842148 hdfs://localhost:37823/backupUT/backup_1696249842148/default/test-1696249791655/ 2023-10-02T12:30:52,683 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-10-02T12:30:52,687 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable table1 2023-10-02T12:30:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=75, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-10-02T12:30:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-10-02T12:30:52,692 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249852692"}]},"ts":"1696249852692"} 2023-10-02T12:30:52,694 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-10-02T12:30:52,789 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-10-02T12:30:52,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=eadca8e9e0b0e864a1bf0e4e75a7b34c, UNASSIGN}] 2023-10-02T12:30:52,792 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=eadca8e9e0b0e864a1bf0e4e75a7b34c, UNASSIGN 2023-10-02T12:30:52,793 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=eadca8e9e0b0e864a1bf0e4e75a7b34c, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:52,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-10-02T12:30:52,794 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; CloseRegionProcedure eadca8e9e0b0e864a1bf0e4e75a7b34c, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:30:52,946 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(119): Close eadca8e9e0b0e864a1bf0e4e75a7b34c 2023-10-02T12:30:52,946 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1668): Closing eadca8e9e0b0e864a1bf0e4e75a7b34c, disabling compactions & flushes 2023-10-02T12:30:52,946 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1690): Closing region table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c. 2023-10-02T12:30:52,946 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c. 2023-10-02T12:30:52,946 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1778): Acquired close lock on table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c. after waiting 0 ms 2023-10-02T12:30:52,946 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1788): Updates disabled for region table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c. 2023-10-02T12:30:52,953 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/eadca8e9e0b0e864a1bf0e4e75a7b34c/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-10-02T12:30:52,953 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:30:52,953 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1909): Closed table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c. 2023-10-02T12:30:52,954 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1622): Region close journal for eadca8e9e0b0e864a1bf0e4e75a7b34c: 2023-10-02T12:30:52,955 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(163): Closed eadca8e9e0b0e864a1bf0e4e75a7b34c 2023-10-02T12:30:52,956 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=eadca8e9e0b0e864a1bf0e4e75a7b34c, regionState=CLOSED 2023-10-02T12:30:52,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=77, resume processing ppid=76 2023-10-02T12:30:52,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=77, ppid=76, state=SUCCESS; CloseRegionProcedure eadca8e9e0b0e864a1bf0e4e75a7b34c, server=jenkins-hbase3.apache.org,38255,1696249782114 in 163 msec 2023-10-02T12:30:52,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=76, resume processing ppid=75 2023-10-02T12:30:52,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=76, ppid=75, state=SUCCESS; TransitRegionStateProcedure table=table1, region=eadca8e9e0b0e864a1bf0e4e75a7b34c, UNASSIGN in 169 msec 2023-10-02T12:30:52,961 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249852960"}]},"ts":"1696249852960"} 2023-10-02T12:30:52,962 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-10-02T12:30:52,964 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-10-02T12:30:52,966 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=75, state=SUCCESS; DisableTableProcedure table=table1 in 278 msec 2023-10-02T12:30:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=75 2023-10-02T12:30:53,002 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete table1 2023-10-02T12:30:53,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-10-02T12:30:53,015 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-10-02T12:30:53,017 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=78, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-10-02T12:30:53,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-10-02T12:30:53,019 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/eadca8e9e0b0e864a1bf0e4e75a7b34c 2023-10-02T12:30:53,021 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/eadca8e9e0b0e864a1bf0e4e75a7b34c/f, FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/eadca8e9e0b0e864a1bf0e4e75a7b34c/recovered.edits] 2023-10-02T12:30:53,026 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/eadca8e9e0b0e864a1bf0e4e75a7b34c/f/e1d13a33e2c54e2db1e896f298b76aec_SeqId_4_ to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/archive/data/default/table1/eadca8e9e0b0e864a1bf0e4e75a7b34c/f/e1d13a33e2c54e2db1e896f298b76aec_SeqId_4_ 2023-10-02T12:30:53,029 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/eadca8e9e0b0e864a1bf0e4e75a7b34c/recovered.edits/6.seqid to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/archive/data/default/table1/eadca8e9e0b0e864a1bf0e4e75a7b34c/recovered.edits/6.seqid 2023-10-02T12:30:53,030 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/eadca8e9e0b0e864a1bf0e4e75a7b34c 2023-10-02T12:30:53,030 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-10-02T12:30:53,033 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=78, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-10-02T12:30:53,035 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-10-02T12:30:53,037 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-10-02T12:30:53,038 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=78, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-10-02T12:30:53,038 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-10-02T12:30:53,039 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696249853038"}]},"ts":"9223372036854775807"} 2023-10-02T12:30:53,040 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:30:53,041 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => eadca8e9e0b0e864a1bf0e4e75a7b34c, NAME => 'table1,,1696249849032.eadca8e9e0b0e864a1bf0e4e75a7b34c.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:30:53,041 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-10-02T12:30:53,041 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1696249853041"}]},"ts":"9223372036854775807"} 2023-10-02T12:30:53,050 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-10-02T12:30:53,052 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(127): Finished pid=78, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-10-02T12:30:53,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=78, state=SUCCESS; DeleteTableProcedure table=table1 in 48 msec 2023-10-02T12:30:53,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=78 2023-10-02T12:30:53,137 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingle Thread=597 (was 600), OpenFileDescriptor=681 (was 696), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=646 (was 648), ProcessCount=181 (was 181), AvailableMemoryMB=2249 (was 3630) 2023-10-02T12:30:53,137 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=597 is superior to 500 2023-10-02T12:30:53,156 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleOverwriteCommand Thread=597, OpenFileDescriptor=681, MaxFileDescriptor=60000, SystemLoadAverage=646, ProcessCount=181, AvailableMemoryMB=2248 2023-10-02T12:30:53,156 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=597 is superior to 500 2023-10-02T12:30:53,156 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestFullRestore(192): test full restore on a single table empty table: command-line 2023-10-02T12:30:53,162 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27e2d551, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:53,165 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:53,166 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43318, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:53,168 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:30:53,169 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36858, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:30:53,182 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:30:53,188 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:30:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249853188 (current time:1696249853188). 2023-10-02T12:30:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:30:53,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:30:53,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fb94e50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:53,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:53,200 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43328, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:53,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:53,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:30:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=79, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:30:53,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-10-02T12:30:53,204 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:30:53,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-10-02T12:30:53,204 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:30:53,207 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:30:53,209 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:30:53,214 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:30:53,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987}] 2023-10-02T12:30:53,216 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=80, ppid=79, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:30:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-10-02T12:30:53,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=80 2023-10-02T12:30:53,367 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:30:53,368 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegion(2819): Flushing 7ec576d712d0f3b0c0592584b88ce987 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-10-02T12:30:53,374 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5 is 169, key is trslm:hdfs://localhost:37823/backupUT\x00test-1696249791655/meta:log-roll-map/1696249848496/Put/seqid=0 2023-10-02T12:30:53,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-10-02T12:30:53,781 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5 2023-10-02T12:30:53,800 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/fe4c59a0e2584ff5980ec23dd90e1c0b is 452, key is session:backup_1696249842148/session:context/1696249848935/Put/seqid=0 2023-10-02T12:30:53,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-10-02T12:30:53,810 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/fe4c59a0e2584ff5980ec23dd90e1c0b 2023-10-02T12:30:53,818 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5 2023-10-02T12:30:53,825 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5, entries=4, sequenceid=61, filesize=5.5 K 2023-10-02T12:30:53,826 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/fe4c59a0e2584ff5980ec23dd90e1c0b as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/fe4c59a0e2584ff5980ec23dd90e1c0b 2023-10-02T12:30:53,831 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/fe4c59a0e2584ff5980ec23dd90e1c0b, entries=2, sequenceid=61, filesize=5.4 K 2023-10-02T12:30:53,832 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 7ec576d712d0f3b0c0592584b88ce987 in 465ms, sequenceid=61, compaction requested=true 2023-10-02T12:30:53,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegion(2520): Flush status journal for 7ec576d712d0f3b0c0592584b88ce987: 2023-10-02T12:30:53,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. for snapshot_backup_system completed. 2023-10-02T12:30:53,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:30:53,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:30:53,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/85d2568c07cc4be686b8e706b89254ce, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5] hfiles 2023-10-02T12:30:53,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98 for snapshot=snapshot_backup_system 2023-10-02T12:30:53,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255 for snapshot=snapshot_backup_system 2023-10-02T12:30:53,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8 for snapshot=snapshot_backup_system 2023-10-02T12:30:53,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/85d2568c07cc4be686b8e706b89254ce for snapshot=snapshot_backup_system 2023-10-02T12:30:53,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5 for snapshot=snapshot_backup_system 2023-10-02T12:30:53,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/f8db732eb5cd417eaa236f25fd437245, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/fe4c59a0e2584ff5980ec23dd90e1c0b] hfiles 2023-10-02T12:30:53,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0 for snapshot=snapshot_backup_system 2023-10-02T12:30:53,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c for snapshot=snapshot_backup_system 2023-10-02T12:30:53,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229 for snapshot=snapshot_backup_system 2023-10-02T12:30:53,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423 for snapshot=snapshot_backup_system 2023-10-02T12:30:53,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/f8db732eb5cd417eaa236f25fd437245 for snapshot=snapshot_backup_system 2023-10-02T12:30:53,832 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/fe4c59a0e2584ff5980ec23dd90e1c0b for snapshot=snapshot_backup_system 2023-10-02T12:30:54,238 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:30:54,238 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=80 2023-10-02T12:30:54,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=80 2023-10-02T12:30:54,239 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:30:54,241 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=80, ppid=79, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:30:54,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=80, resume processing ppid=79 2023-10-02T12:30:54,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=80, ppid=79, state=SUCCESS; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 in 1.0280 sec 2023-10-02T12:30:54,244 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:30:54,245 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:30:54,246 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:30:54,246 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:30:54,247 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:30:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-10-02T12:30:54,676 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:30:54,688 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:30:54,689 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:30:54,690 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:30:54,690 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-10-02T12:30:54,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.4900 sec 2023-10-02T12:30:55,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=79 2023-10-02T12:30:55,311 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(115): Backup backup_1696249853162 started at 1696249855311. 2023-10-02T12:30:55,313 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(119): Backup session backup_1696249853162 has been started. 2023-10-02T12:30:55,318 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:30:55,318 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:30:55,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:30:55,319 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:30:55,319 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:30:55,320 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:30:55,320 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:30:55,320 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:55,330 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:55,330 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:30:55,330 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:30:55,330 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:30:55,331 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:30:55,331 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:55,331 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:30:55,331 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:30:55,331 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:30:55,332 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:30:55,332 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:30:55,332 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:30:55,332 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:30:55,333 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:30:55,333 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:30:55,333 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:30:55,335 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:55,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:55,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:30:55,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:30:55,335 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:30:55,335 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:30:55,336 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:30:55,336 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:55,336 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:55,336 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:30:55,337 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:30:55,337 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@127a793c[Count = 0] remaining members to acquire global barrier 2023-10-02T12:30:55,337 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:30:55,337 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:55,338 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:55,338 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:30:55,338 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:30:55,338 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:30:55,338 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:30:55,339 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:55,339 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696249844439 highest: 1696249844439 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:55,339 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114:(num 1696249844439) roll requested 2023-10-02T12:30:55,356 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249855339, exclude list is [], retry=0 2023-10-02T12:30:55,364 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:30:55,366 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249844439 with entries=13, filesize=3.09 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249855339 2023-10-02T12:30:55,366 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:30:55,366 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249844439 is not closed yet, will try archiving it next time 2023-10-02T12:30:55,366 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta:.meta(num 1696249844458) roll requested 2023-10-02T12:30:55,366 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249837258 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/oldWALs/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249837258 2023-10-02T12:30:55,384 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249855366.meta, exclude list is [], retry=0 2023-10-02T12:30:55,389 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:30:55,391 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249844458.meta with entries=12, filesize=2.73 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249855366.meta 2023-10-02T12:30:55,391 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:30:55,392 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249844458.meta is not closed yet, will try archiving it next time 2023-10-02T12:30:55,399 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2023-10-02T12:30:55,399 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696249855339 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:55,409 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:30:55,409 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,38255,1696249782114' in zk 2023-10-02T12:30:55,413 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:55,413 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:30:55,413 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:30:55,413 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:30:55,413 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:30:55,413 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:30:55,415 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:30:55,415 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:30:55,416 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:55,416 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:55,416 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:30:55,417 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:55,417 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:55,418 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,38255,1696249782114': 2023-10-02T12:30:55,418 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:30:55,418 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:30:55,418 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:30:55,418 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:30:55,418 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:30:55,418 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:30:55,420 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:55,420 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:30:55,421 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:30:55,421 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:30:55,421 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:55,421 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:30:55,421 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:30:55,421 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:30:55,421 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:30:55,422 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:55,422 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:30:55,423 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:55,423 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:55,424 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:30:55,424 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:30:55,424 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:55,429 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:30:55,429 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:30:55,429 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:30:55,429 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:30:55,430 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:30:55,430 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:30:55,430 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:30:55,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:30:55,562 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249855552_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } 2023-10-02T12:30:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249855562 (current time:1696249855562). 2023-10-02T12:30:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:30:55,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249855552_default_test-1696249791655 VERSION not specified, setting to 2 2023-10-02T12:30:55,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663c1039, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:55,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:55,603 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43342, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:30:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=81, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1696249855552_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } 2023-10-02T12:30:55,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249855552_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 }, snapshot procedure id = 81 2023-10-02T12:30:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-10-02T12:30:55,613 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1696249855552_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:30:55,618 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1696249855552_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:30:55,629 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249855552_default_test-1696249791655 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:30:55,631 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1696249855552_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:30:55,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-10-02T12:30:55,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-10-02T12:30:56,038 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1696249855552_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:30:56,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE; SnapshotRegionProcedure c8f99eaa6d1b90df2e9c811158390413}] 2023-10-02T12:30:56,040 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=82, ppid=81, state=RUNNABLE; SnapshotRegionProcedure c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:56,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=82 2023-10-02T12:30:56,191 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. 2023-10-02T12:30:56,191 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.HRegion(2520): Flush status journal for c8f99eaa6d1b90df2e9c811158390413: 2023-10-02T12:30:56,191 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. for snapshot_1696249855552_default_test-1696249791655 completed. 2023-10-02T12:30:56,191 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(240): Storing 'test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413.' region-info for snapshot=snapshot_1696249855552_default_test-1696249791655 2023-10-02T12:30:56,191 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:30:56,191 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_] hfiles 2023-10-02T12:30:56,191 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ for snapshot=snapshot_1696249855552_default_test-1696249791655 2023-10-02T12:30:56,197 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. 2023-10-02T12:30:56,197 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=82 2023-10-02T12:30:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=82 2023-10-02T12:30:56,197 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249855552_default_test-1696249791655 on region c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:56,199 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=82, ppid=81, state=RUNNABLE; SnapshotRegionProcedure c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:30:56,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=82, resume processing ppid=81 2023-10-02T12:30:56,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=82, ppid=81, state=SUCCESS; SnapshotRegionProcedure c8f99eaa6d1b90df2e9c811158390413 in 161 msec 2023-10-02T12:30:56,201 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1696249855552_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:30:56,201 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1696249855552_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:30:56,202 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1696249855552_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:30:56,202 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249855552_default_test-1696249791655 2023-10-02T12:30:56,202 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249855552_default_test-1696249791655 2023-10-02T12:30:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-10-02T12:30:56,219 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1696249855552_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:30:56,224 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1696249855552_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:30:56,225 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249855552_default_test-1696249791655 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249855552_default_test-1696249791655 2023-10-02T12:30:56,226 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1696249855552_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:30:56,226 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249855552_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 }, snapshot procedure id = 81 2023-10-02T12:30:56,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=81, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1696249855552_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } in 621 msec 2023-10-02T12:30:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=81 2023-10-02T12:30:56,719 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696249853162 2023-10-02T12:30:56,719 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:30:56,719 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-10-02T12:30:56,719 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249853162_test-1696249791655 2023-10-02T12:30:56,719 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249855552_default_test-1696249791655 to hdfs://localhost:37823/backupUT/backup_1696249853162/default/test-1696249791655/ 2023-10-02T12:30:56,719 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:30:56,743 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:30:56,743 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,40619,1696249778965_1578017906_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37823/backupUT/backup_1696249853162/default/test-1696249791655, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37823/backupUT/backup_1696249853162/default/test-1696249791655/.hbase-snapshot/.tmp/snapshot_1696249855552_default_test-1696249791655 2023-10-02T12:30:56,743 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07, snapshotDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249855552_default_test-1696249791655. 2023-10-02T12:30:56,752 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249855552_default_test-1696249791655 to hdfs://localhost:37823/backupUT/backup_1696249853162/default/test-1696249791655/.hbase-snapshot/.tmp/snapshot_1696249855552_default_test-1696249791655 2023-10-02T12:30:57,177 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:57,178 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:57,178 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:57,179 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:58,462 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-9054563253316719792.jar 2023-10-02T12:30:58,463 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:58,463 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:58,464 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:58,464 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:58,465 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:58,465 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:58,466 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:58,466 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:58,467 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:30:58,468 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:30:58,469 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:30:58,469 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:30:58,470 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:30:58,471 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:30:58,471 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:30:58,472 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:30:58,472 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:30:58,473 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:30:58,473 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:30:58,474 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:30:58,475 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:30:58,475 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:58,476 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:58,477 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:58,477 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:58,478 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:30:58,479 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:58,480 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:30:58,541 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249855552_default_test-1696249791655' hfile list 2023-10-02T12:30:58,544 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-10-02T12:30:58,649 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249858600/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:30:58,649 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249858600/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:30:58,733 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:30:58,759 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ output=hdfs://localhost:37823/backupUT/backup_1696249853162/default/test-1696249791655/archive/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ 2023-10-02T12:30:58,759 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 4.160M/sec 2023-10-02T12:30:59,697 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:30:59,699 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:30:59,727 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249855552_default_test-1696249791655 2023-10-02T12:30:59,727 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249855552_default_test-1696249791655 finished. 2023-10-02T12:30:59,727 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1034): test-1696249791655 2023-10-02T12:30:59,760 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249853162 2023-10-02T12:30:59,760 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:30:59,814 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37823/backupUT/backup_1696249853162/.backup.manifest 2023-10-02T12:30:59,814 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(358): Backup backup_1696249853162 finished: type=FULL,tablelist=test-1696249791655,targetRootDir=hdfs://localhost:37823/backupUT,startts=1696249855311,completets=1696249859760,bytescopied=0 2023-10-02T12:30:59,815 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:30:59,815 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249855552_default_test-1696249791655 2023-10-02T12:30:59,816 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249855552_default_test-1696249791655" 2023-10-02T12:30:59,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249855552_default_test-1696249791655 2023-10-02T12:30:59,821 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249855552_default_test-1696249791655 for backup backup_1696249853162 succeeded. 2023-10-02T12:30:59,822 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:30:59,829 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:30:59,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:30:59,833 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:30:59,836 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:30:59,844 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(376): Backup backup_1696249853162 completed. 2023-10-02T12:30:59,845 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:30:59,870 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestFullRestore(196): backup complete 2023-10-02T12:30:59,871 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] backup.RestoreDriver(101): Found -overwrite option in restore command, will overwrite to existing table if any in the restore target 2023-10-02T12:30:59,876 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1513397c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:30:59,878 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:30:59,880 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:43350, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:30:59,883 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249853162/.backup.manifest 2023-10-02T12:30:59,887 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249853162/.backup.manifest 2023-10-02T12:30:59,888 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249853162/.backup.manifest 2023-10-02T12:30:59,890 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249853162/.backup.manifest 2023-10-02T12:30:59,891 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(148): Restoring 'test-1696249791655' to 'test-1696249791655' from full backup image hdfs://localhost:37823/backupUT/backup_1696249853162/default/test-1696249791655 2023-10-02T12:30:59,916 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(476): Truncating exising target table 'test-1696249791655', preserving region splits 2023-10-02T12:30:59,922 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:30:59,924 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:36860, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:30:59,924 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable test-1696249791655 2023-10-02T12:30:59,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=83, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=test-1696249791655 2023-10-02T12:30:59,930 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696249791655","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249859930"}]},"ts":"1696249859930"} 2023-10-02T12:30:59,935 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696249791655, state=DISABLING in hbase:meta 2023-10-02T12:30:59,937 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set test-1696249791655 to state=DISABLING 2023-10-02T12:30:59,938 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1696249791655, region=c8f99eaa6d1b90df2e9c811158390413, UNASSIGN}] 2023-10-02T12:30:59,939 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1696249791655, region=c8f99eaa6d1b90df2e9c811158390413, UNASSIGN 2023-10-02T12:30:59,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=83 2023-10-02T12:30:59,940 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=84 updating hbase:meta row=c8f99eaa6d1b90df2e9c811158390413, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:30:59,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE; CloseRegionProcedure c8f99eaa6d1b90df2e9c811158390413, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:31:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=83 2023-10-02T12:31:00,103 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(119): Close c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:31:00,103 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1668): Closing c8f99eaa6d1b90df2e9c811158390413, disabling compactions & flushes 2023-10-02T12:31:00,103 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1690): Closing region test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. 2023-10-02T12:31:00,103 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. 2023-10-02T12:31:00,103 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1778): Acquired close lock on test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. after waiting 0 ms 2023-10-02T12:31:00,103 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1788): Updates disabled for region test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. 2023-10-02T12:31:00,118 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/recovered.edits/10.seqid, newMaxSeqId=10, maxSeqId=1 2023-10-02T12:31:00,122 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:31:00,124 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1909): Closed test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413. 2023-10-02T12:31:00,124 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1622): Region close journal for c8f99eaa6d1b90df2e9c811158390413: 2023-10-02T12:31:00,132 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(163): Closed c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:31:00,132 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=84 updating hbase:meta row=c8f99eaa6d1b90df2e9c811158390413, regionState=CLOSED 2023-10-02T12:31:00,134 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1696249791655 2023-10-02T12:31:00,134 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-10-02T12:31:00,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=85, resume processing ppid=84 2023-10-02T12:31:00,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=85, ppid=84, state=SUCCESS; CloseRegionProcedure c8f99eaa6d1b90df2e9c811158390413, server=jenkins-hbase3.apache.org,38255,1696249782114 in 186 msec 2023-10-02T12:31:00,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=84, resume processing ppid=83 2023-10-02T12:31:00,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=84, ppid=83, state=SUCCESS; TransitRegionStateProcedure table=test-1696249791655, region=c8f99eaa6d1b90df2e9c811158390413, UNASSIGN in 198 msec 2023-10-02T12:31:00,139 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696249791655","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249860138"}]},"ts":"1696249860138"} 2023-10-02T12:31:00,140 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696249791655, state=DISABLED in hbase:meta 2023-10-02T12:31:00,142 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set test-1696249791655 to state=DISABLED 2023-10-02T12:31:00,144 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=83, state=SUCCESS; DisableTableProcedure table=test-1696249791655 in 219 msec 2023-10-02T12:31:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=83 2023-10-02T12:31:00,251 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$6(2480): Client=jenkins//172.31.12.81 truncate test-1696249791655 2023-10-02T12:31:00,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=86, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=test-1696249791655 preserveSplits=true) 2023-10-02T12:31:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=86 2023-10-02T12:31:00,256 DEBUG [PEWorker-4 {}] procedure.TruncateTableProcedure(87): waiting for 'test-1696249791655' regions in transition 2023-10-02T12:31:00,260 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:31:00,262 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f, FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/recovered.edits] 2023-10-02T12:31:00,270 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/archive/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ 2023-10-02T12:31:00,274 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/recovered.edits/10.seqid to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/archive/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/recovered.edits/10.seqid 2023-10-02T12:31:00,275 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:31:00,275 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(293): Archived test-1696249791655 regions 2023-10-02T12:31:00,281 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of test-1696249791655 from hbase:meta 2023-10-02T12:31:00,285 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(387): Removing 'test-1696249791655' descriptor. 2023-10-02T12:31:00,287 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(377): Removing 'test-1696249791655' from region states. 2023-10-02T12:31:00,287 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696249860287"}]},"ts":"9223372036854775807"} 2023-10-02T12:31:00,295 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:31:00,295 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => c8f99eaa6d1b90df2e9c811158390413, NAME => 'test-1696249791655,,1696249824014.c8f99eaa6d1b90df2e9c811158390413.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:31:00,295 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(381): Marking 'test-1696249791655' as deleted. 2023-10-02T12:31:00,295 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1696249791655","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1696249860295"}]},"ts":"9223372036854775807"} 2023-10-02T12:31:00,298 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1678): Deleted table test-1696249791655 state from META 2023-10-02T12:31:00,318 INFO [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 35dcfa6f2558560536edbd70539305b1, NAME => 'test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1696249791655', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:31:00,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=86 2023-10-02T12:31:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=86 2023-10-02T12:31:00,750 DEBUG [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(888): Instantiated test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:31:00,750 DEBUG [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1668): Closing 35dcfa6f2558560536edbd70539305b1, disabling compactions & flushes 2023-10-02T12:31:00,750 INFO [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1690): Closing region test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1. 2023-10-02T12:31:00,750 DEBUG [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1. 2023-10-02T12:31:00,750 DEBUG [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1. after waiting 0 ms 2023-10-02T12:31:00,750 DEBUG [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1. 2023-10-02T12:31:00,750 INFO [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1909): Closed test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1. 2023-10-02T12:31:00,750 DEBUG [RegionOpenAndInit-test-1696249791655-pool-0 {}] regionserver.HRegion(1622): Region close journal for 35dcfa6f2558560536edbd70539305b1: 2023-10-02T12:31:00,752 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1696249860752"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249860752"}]},"ts":"1696249860752"} 2023-10-02T12:31:00,754 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:31:00,755 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696249791655","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249860755"}]},"ts":"1696249860755"} 2023-10-02T12:31:00,757 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696249791655, state=ENABLING in hbase:meta 2023-10-02T12:31:00,782 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1696249791655, region=35dcfa6f2558560536edbd70539305b1, ASSIGN}] 2023-10-02T12:31:00,784 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=87, ppid=86, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1696249791655, region=35dcfa6f2558560536edbd70539305b1, ASSIGN 2023-10-02T12:31:00,785 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=87, ppid=86, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1696249791655, region=35dcfa6f2558560536edbd70539305b1, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38255,1696249782114; forceNewPlan=false, retain=false 2023-10-02T12:31:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=86 2023-10-02T12:31:00,938 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=87 updating hbase:meta row=35dcfa6f2558560536edbd70539305b1, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:00,940 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE; OpenRegionProcedure 35dcfa6f2558560536edbd70539305b1, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:31:01,096 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(130): Open test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1. 2023-10-02T12:31:01,096 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7254): Opening region: {ENCODED => 35dcfa6f2558560536edbd70539305b1, NAME => 'test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:31:01,097 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:31:01,097 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1696249791655 35dcfa6f2558560536edbd70539305b1 2023-10-02T12:31:01,097 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(888): Instantiated test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:31:01,097 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7296): checking encryption for 35dcfa6f2558560536edbd70539305b1 2023-10-02T12:31:01,097 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7299): checking classloading for 35dcfa6f2558560536edbd70539305b1 2023-10-02T12:31:01,099 INFO [StoreOpener-35dcfa6f2558560536edbd70539305b1-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 35dcfa6f2558560536edbd70539305b1 2023-10-02T12:31:01,100 INFO [StoreOpener-35dcfa6f2558560536edbd70539305b1-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35dcfa6f2558560536edbd70539305b1 columnFamilyName f 2023-10-02T12:31:01,100 DEBUG [StoreOpener-35dcfa6f2558560536edbd70539305b1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:31:01,101 INFO [StoreOpener-35dcfa6f2558560536edbd70539305b1-1 {}] regionserver.HStore(324): Store=35dcfa6f2558560536edbd70539305b1/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:31:01,102 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1 2023-10-02T12:31:01,102 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1 2023-10-02T12:31:01,105 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1076): writing seq id for 35dcfa6f2558560536edbd70539305b1 2023-10-02T12:31:01,110 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:31:01,111 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): Opened 35dcfa6f2558560536edbd70539305b1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9568414560, jitterRate=-0.1088719516992569}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:31:01,112 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(992): Region open journal for 35dcfa6f2558560536edbd70539305b1: 2023-10-02T12:31:01,113 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1., pid=88, masterSystemTime=1696249861092 2023-10-02T12:31:01,114 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1. 2023-10-02T12:31:01,115 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(158): Opened test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1. 2023-10-02T12:31:01,115 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=87 updating hbase:meta row=35dcfa6f2558560536edbd70539305b1, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:01,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=88, resume processing ppid=87 2023-10-02T12:31:01,118 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=88, ppid=87, state=SUCCESS; OpenRegionProcedure 35dcfa6f2558560536edbd70539305b1, server=jenkins-hbase3.apache.org,38255,1696249782114 in 176 msec 2023-10-02T12:31:01,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=87, resume processing ppid=86 2023-10-02T12:31:01,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=87, ppid=86, state=SUCCESS; TransitRegionStateProcedure table=test-1696249791655, region=35dcfa6f2558560536edbd70539305b1, ASSIGN in 336 msec 2023-10-02T12:31:01,121 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1696249791655","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249861121"}]},"ts":"1696249861121"} 2023-10-02T12:31:01,122 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1696249791655, state=ENABLED in hbase:meta 2023-10-02T12:31:01,124 DEBUG [PEWorker-4 {}] procedure.TruncateTableProcedure(145): truncate 'test-1696249791655' completed 2023-10-02T12:31:01,126 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=86, state=SUCCESS; TruncateTableProcedure (table=test-1696249791655 preserveSplits=true) in 872 msec 2023-10-02T12:31:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=86 2023-10-02T12:31:01,371 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:37823/backupUT/backup_1696249853162/default/test-1696249791655/archive/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413 from hbase tables test-1696249791655 to tables test-1696249791655 2023-10-02T12:31:01,371 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1696249791655 into test-1696249791655 2023-10-02T12:31:01,373 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-test-1696249791655-1696249861371 from hdfs://localhost:37823/backupUT/backup_1696249853162/default/test-1696249791655/archive/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413 2023-10-02T12:31:01,379 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3af71f66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:01,380 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:31:01,382 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60778, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:31:01,384 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-10-02T12:31:01,384 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1696249791655 2023-10-02T12:31:01,395 DEBUG [hconnection-0x5a489ce9-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:01,396 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49344, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:01,398 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1696249791655: 2023-10-02T12:31:01,398 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-10-02T12:31:01,398 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/partitions_e4d46bf7-4bf0-4043-81b1-35288639eeb9 2023-10-02T12:31:01,406 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:01,407 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:01,408 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:01,408 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:02,447 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-2654340992238696514.jar 2023-10-02T12:31:02,448 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:02,448 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:02,448 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:02,449 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:02,449 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:02,449 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:02,450 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:02,450 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:02,451 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:02,452 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:31:02,453 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:31:02,454 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:31:02,454 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:31:02,455 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:31:02,456 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:31:02,456 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:31:02,457 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:31:02,458 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:31:02,458 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:31:02,459 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:31:02,459 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:31:02,461 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:02,461 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:02,462 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:02,462 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:02,463 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:02,463 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:02,464 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:02,464 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: test-1696249791655 2023-10-02T12:31:02,465 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:02,465 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-10-02T12:31:02,465 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-10-02T12:31:02,466 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:31:02,480 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-10-02T12:31:02,690 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249862573/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:31:02,690 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249862573/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:31:02,798 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:37823/backupUT/backup_1696249853162/default/test-1696249791655/archive/data/default/test-1696249791655/c8f99eaa6d1b90df2e9c811158390413/f/14fdcd6ee6a9491ea7c88a42ae025e37_SeqId_4_ 2023-10-02T12:31:02,809 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-10-02T12:31:02,858 DEBUG [pool-331-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e06ef50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:02,865 DEBUG [hconnection-0x51c73bcf-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:02,867 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49346, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:02,874 DEBUG [pool-331-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:02,874 DEBUG [pool-331-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-10-02T12:31:02,874 DEBUG [pool-331-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-10-02T12:31:02,896 DEBUG [pool-331-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-test-1696249791655-1696249861371/_temporary/0/_temporary/attempt_local127376466_0011_r_000000_0/f/6d29ebd5b7344cf49b17a419461915a9 is 37, key is row10/f:q1/1696249794327/Put/seqid=0 2023-10-02T12:31:02,898 WARN [Thread-1395 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:38255] Specified favored nodes: [jenkins-hbase3.apache.org:38255] 2023-10-02T12:31:03,743 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-test-1696249791655-1696249861371 2023-10-02T12:31:03,756 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e14bcb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:03,759 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:03,760 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49362, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:03,775 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-test-1696249791655-1696249861371/_SUCCESS 2023-10-02T12:31:03,783 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:31:03,784 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60794, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:31:03,801 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-test-1696249791655-1696249861371/f/6d29ebd5b7344cf49b17a419461915a9 first=Optional[row0] last=Optional[row98] 2023-10-02T12:31:03,803 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1., hostname=jenkins-hbase3.apache.org,38255,1696249782114, seqNum=2 for row with hfile group [{f,hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-test-1696249791655-1696249861371/f/6d29ebd5b7344cf49b17a419461915a9}] 2023-10-02T12:31:03,821 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-test-1696249791655-1696249861371/f/6d29ebd5b7344cf49b17a419461915a9 for inclusion in 35dcfa6f2558560536edbd70539305b1/f 2023-10-02T12:31:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-10-02T12:31:03,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(625): Region bounds: first= last= 2023-10-02T12:31:03,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HRegion(2520): Flush status journal for 35dcfa6f2558560536edbd70539305b1: 2023-10-02T12:31:03,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-test-1696249791655-1696249861371/f/6d29ebd5b7344cf49b17a419461915a9 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__test-1696249791655__3sneulgvhj5ggo9hrv3vka7ej0j8rnetr7n9s7bkoook4na01ugt2o5s2c4jgsda/f/6d29ebd5b7344cf49b17a419461915a9 2023-10-02T12:31:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__test-1696249791655__3sneulgvhj5ggo9hrv3vka7ej0j8rnetr7n9s7bkoook4na01ugt2o5s2c4jgsda/f/6d29ebd5b7344cf49b17a419461915a9 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1/f/553dd16623454c9282013d29ef554af3_SeqId_4_ 2023-10-02T12:31:03,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59fb325d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:03,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:31:03,839 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60800, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:31:03,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:03,847 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49370, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(428): write bulk load descriptor to backup test-1696249791655 with 1 entries 2023-10-02T12:31:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(1602): writing raw bulk path hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1/f/553dd16623454c9282013d29ef554af3_SeqId_4_ for test-1696249791655 35dcfa6f2558560536edbd70539305b1 2023-10-02T12:31:03,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(435): written 1 rows for bulk load of test-1696249791655 2023-10-02T12:31:03,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:03,878 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__test-1696249791655__3sneulgvhj5ggo9hrv3vka7ej0j8rnetr7n9s7bkoook4na01ugt2o5s2c4jgsda/f/6d29ebd5b7344cf49b17a419461915a9 into 35dcfa6f2558560536edbd70539305b1/f as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1/f/553dd16623454c9282013d29ef554af3_SeqId_4_ - updating store file list. 2023-10-02T12:31:03,884 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1/f/553dd16623454c9282013d29ef554af3_SeqId_4_ into 35dcfa6f2558560536edbd70539305b1/f 2023-10-02T12:31:03,884 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__test-1696249791655__3sneulgvhj5ggo9hrv3vka7ej0j8rnetr7n9s7bkoook4na01ugt2o5s2c4jgsda/f/6d29ebd5b7344cf49b17a419461915a9 into 35dcfa6f2558560536edbd70539305b1/f (new location: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1/f/553dd16623454c9282013d29ef554af3_SeqId_4_) 2023-10-02T12:31:03,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__test-1696249791655__3sneulgvhj5ggo9hrv3vka7ej0j8rnetr7n9s7bkoook4na01ugt2o5s2c4jgsda/f/6d29ebd5b7344cf49b17a419461915a9 2023-10-02T12:31:03,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b72ed00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:03,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:31:03,987 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60812, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-10-02T12:31:03,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:03,996 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49380, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-10-02T12:31:04,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup test-1696249791655 with 1 entries 2023-10-02T12:31:04,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(1523): writing done bulk path hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1/f/553dd16623454c9282013d29ef554af3_SeqId_4_ for test-1696249791655 35dcfa6f2558560536edbd70539305b1 2023-10-02T12:31:04,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(414): written 1 rows for bulk load of test-1696249791655 2023-10-02T12:31:04,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:04,050 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:04,050 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-10-02T12:31:04,050 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-10-02T12:31:04,050 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(233): Backup: backup_1696249853162 hdfs://localhost:37823/backupUT/backup_1696249853162/default/test-1696249791655/ 2023-10-02T12:31:04,051 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-10-02T12:31:04,051 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:04,082 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleOverwriteCommand Thread=602 (was 597) - Thread LEAK? -, OpenFileDescriptor=695 (was 681) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=631 (was 646), ProcessCount=181 (was 181), AvailableMemoryMB=2046 (was 2248) 2023-10-02T12:31:04,082 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=602 is superior to 500 2023-10-02T12:31:04,107 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleDNECommand Thread=602, OpenFileDescriptor=695, MaxFileDescriptor=60000, SystemLoadAverage=631, ProcessCount=181, AvailableMemoryMB=2045 2023-10-02T12:31:04,107 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=602 is superior to 500 2023-10-02T12:31:04,107 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestFullRestore(322): test restore fails on multiple tables that do not exist: command-line 2023-10-02T12:31:04,116 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11cf5304, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:04,127 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:04,128 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49396, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:04,131 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:31:04,132 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60820, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:31:04,151 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:31:04,157 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:31:04,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249864157 (current time:1696249864157). 2023-10-02T12:31:04,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:31:04,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:31:04,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672701f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:04,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:04,166 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49398, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:04,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:04,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:31:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=89, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:31:04,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 89 2023-10-02T12:31:04,171 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:31:04,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-10-02T12:31:04,172 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:31:04,176 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:31:04,179 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:31:04,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-10-02T12:31:04,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-10-02T12:31:04,587 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:31:04,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987}] 2023-10-02T12:31:04,589 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=90, ppid=89, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:31:04,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=90 2023-10-02T12:31:04,741 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:31:04,741 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegion(2819): Flushing 7ec576d712d0f3b0c0592584b88ce987 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-10-02T12:31:04,745 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/9bf0c2ed3e5c43ba8bb2287a7e2731fa is 169, key is trslm:hdfs://localhost:37823/backupUT\x00test-1696249791655/meta:log-roll-map/1696249859737/Put/seqid=0 2023-10-02T12:31:04,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-10-02T12:31:05,152 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/9bf0c2ed3e5c43ba8bb2287a7e2731fa 2023-10-02T12:31:05,163 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/2129c391bd7447f0b05fea1084f2d47d is 452, key is session:backup_1696249853162/session:context/1696249859834/Put/seqid=0 2023-10-02T12:31:05,172 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/2129c391bd7447f0b05fea1084f2d47d 2023-10-02T12:31:05,181 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/9bf0c2ed3e5c43ba8bb2287a7e2731fa as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/9bf0c2ed3e5c43ba8bb2287a7e2731fa 2023-10-02T12:31:05,187 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/9bf0c2ed3e5c43ba8bb2287a7e2731fa, entries=4, sequenceid=72, filesize=5.5 K 2023-10-02T12:31:05,188 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/2129c391bd7447f0b05fea1084f2d47d as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/2129c391bd7447f0b05fea1084f2d47d 2023-10-02T12:31:05,193 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/2129c391bd7447f0b05fea1084f2d47d, entries=2, sequenceid=72, filesize=5.4 K 2023-10-02T12:31:05,194 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 7ec576d712d0f3b0c0592584b88ce987 in 453ms, sequenceid=72, compaction requested=true 2023-10-02T12:31:05,194 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegion(2520): Flush status journal for 7ec576d712d0f3b0c0592584b88ce987: 2023-10-02T12:31:05,194 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. for snapshot_backup_system completed. 2023-10-02T12:31:05,194 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:31:05,194 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:31:05,194 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/85d2568c07cc4be686b8e706b89254ce, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/9bf0c2ed3e5c43ba8bb2287a7e2731fa] hfiles 2023-10-02T12:31:05,194 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98 for snapshot=snapshot_backup_system 2023-10-02T12:31:05,194 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255 for snapshot=snapshot_backup_system 2023-10-02T12:31:05,195 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8 for snapshot=snapshot_backup_system 2023-10-02T12:31:05,195 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/85d2568c07cc4be686b8e706b89254ce for snapshot=snapshot_backup_system 2023-10-02T12:31:05,195 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5 for snapshot=snapshot_backup_system 2023-10-02T12:31:05,195 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/9bf0c2ed3e5c43ba8bb2287a7e2731fa for snapshot=snapshot_backup_system 2023-10-02T12:31:05,195 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/f8db732eb5cd417eaa236f25fd437245, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/fe4c59a0e2584ff5980ec23dd90e1c0b, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/2129c391bd7447f0b05fea1084f2d47d] hfiles 2023-10-02T12:31:05,195 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0 for snapshot=snapshot_backup_system 2023-10-02T12:31:05,195 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c for snapshot=snapshot_backup_system 2023-10-02T12:31:05,195 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229 for snapshot=snapshot_backup_system 2023-10-02T12:31:05,195 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423 for snapshot=snapshot_backup_system 2023-10-02T12:31:05,195 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/f8db732eb5cd417eaa236f25fd437245 for snapshot=snapshot_backup_system 2023-10-02T12:31:05,195 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/fe4c59a0e2584ff5980ec23dd90e1c0b for snapshot=snapshot_backup_system 2023-10-02T12:31:05,195 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/2129c391bd7447f0b05fea1084f2d47d for snapshot=snapshot_backup_system 2023-10-02T12:31:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-10-02T12:31:05,601 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:31:05,601 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=90 2023-10-02T12:31:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=90 2023-10-02T12:31:05,602 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:31:05,603 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=90, ppid=89, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:31:05,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=90, resume processing ppid=89 2023-10-02T12:31:05,606 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:31:05,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=90, ppid=89, state=SUCCESS; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 in 1.0170 sec 2023-10-02T12:31:05,613 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:31:05,614 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:31:05,614 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:31:05,615 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:31:05,626 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:31:06,029 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:31:06,040 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:31:06,041 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:31:06,042 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:31:06,042 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 89 2023-10-02T12:31:06,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=89, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8740 sec 2023-10-02T12:31:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=89 2023-10-02T12:31:06,278 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(115): Backup backup_1696249864116 started at 1696249866278. 2023-10-02T12:31:06,280 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(119): Backup session backup_1696249864116 has been started. 2023-10-02T12:31:06,286 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:31:06,287 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:31:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:31:06,287 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:31:06,287 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:31:06,288 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:31:06,288 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:31:06,289 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:31:06,291 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:06,291 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:31:06,291 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:31:06,291 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:31:06,291 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:31:06,292 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:31:06,292 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:31:06,292 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:31:06,292 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:31:06,292 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:31:06,293 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:31:06,293 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:31:06,294 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:31:06,294 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:31:06,294 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:31:06,296 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:31:06,296 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:06,296 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:31:06,296 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:31:06,297 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:31:06,297 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:31:06,297 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:31:06,297 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:06,298 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:06,298 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:31:06,298 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:31:06,298 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@5264ddbf[Count = 0] remaining members to acquire global barrier 2023-10-02T12:31:06,298 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:31:06,298 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:31:06,299 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:31:06,300 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:31:06,300 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:31:06,300 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:31:06,300 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:31:06,300 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:06,300 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696249855339 highest: 1696249855339 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:06,300 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114:(num 1696249855339) roll requested 2023-10-02T12:31:06,312 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249866300, exclude list is [], retry=0 2023-10-02T12:31:06,316 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:31:06,318 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249855339 with entries=15, filesize=4.76 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249866300 2023-10-02T12:31:06,318 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:31:06,318 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249855339 is not closed yet, will try archiving it next time 2023-10-02T12:31:06,318 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta:.meta(num 1696249855366) roll requested 2023-10-02T12:31:06,318 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249844439 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/oldWALs/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249844439 2023-10-02T12:31:06,332 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249866318.meta, exclude list is [], retry=0 2023-10-02T12:31:06,335 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:31:06,337 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249855366.meta with entries=12, filesize=3.07 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249866318.meta 2023-10-02T12:31:06,337 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:31:06,337 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249855366.meta is not closed yet, will try archiving it next time 2023-10-02T12:31:06,340 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-10-02T12:31:06,340 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696249866300 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:06,351 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:31:06,351 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,38255,1696249782114' in zk 2023-10-02T12:31:06,353 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:31:06,353 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:31:06,353 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:31:06,353 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:06,354 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:31:06,355 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:31:06,355 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:31:06,355 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:31:06,355 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:06,356 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:06,356 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:31:06,356 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:06,357 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:06,357 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,38255,1696249782114': 2023-10-02T12:31:06,357 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:31:06,357 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:31:06,357 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:31:06,357 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:31:06,357 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:31:06,357 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:31:06,360 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:31:06,360 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:31:06,360 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:31:06,360 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:31:06,360 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:31:06,361 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:31:06,361 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:31:06,361 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:31:06,361 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:31:06,361 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:06,362 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:31:06,362 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:06,363 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:06,363 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:31:06,364 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:06,364 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:06,368 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:31:06,368 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:31:06,368 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:31:06,368 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:31:06,369 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:31:06,369 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:31:06,369 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:31:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:31:06,502 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249866498_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } 2023-10-02T12:31:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249866502 (current time:1696249866502). 2023-10-02T12:31:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:31:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249866498_ns2_test-16962497916551 VERSION not specified, setting to 2 2023-10-02T12:31:06,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4394b78d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:06,516 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49410, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:31:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=91, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1696249866498_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } 2023-10-02T12:31:06,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249866498_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 }, snapshot procedure id = 91 2023-10-02T12:31:06,525 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1696249866498_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:31:06,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-10-02T12:31:06,526 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1696249866498_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:31:06,529 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249866498_ns2_test-16962497916551 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:31:06,531 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1696249866498_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:31:06,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-10-02T12:31:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-10-02T12:31:06,938 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1696249866498_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:31:06,938 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; SnapshotRegionProcedure ce860946aa8cd08494bb77afea49e9de}] 2023-10-02T12:31:06,940 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=92, ppid=91, state=RUNNABLE; SnapshotRegionProcedure ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:31:07,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=92 2023-10-02T12:31:07,091 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. 2023-10-02T12:31:07,091 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.HRegion(2520): Flush status journal for ce860946aa8cd08494bb77afea49e9de: 2023-10-02T12:31:07,092 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. for snapshot_1696249866498_ns2_test-16962497916551 completed. 2023-10-02T12:31:07,092 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de.' region-info for snapshot=snapshot_1696249866498_ns2_test-16962497916551 2023-10-02T12:31:07,092 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:31:07,092 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_] hfiles 2023-10-02T12:31:07,092 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_ for snapshot=snapshot_1696249866498_ns2_test-16962497916551 2023-10-02T12:31:07,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-10-02T12:31:07,498 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. 2023-10-02T12:31:07,498 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=92 2023-10-02T12:31:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=92 2023-10-02T12:31:07,498 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249866498_ns2_test-16962497916551 on region ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:31:07,500 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=92, ppid=91, state=RUNNABLE; SnapshotRegionProcedure ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:31:07,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=92, resume processing ppid=91 2023-10-02T12:31:07,502 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1696249866498_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:31:07,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=92, ppid=91, state=SUCCESS; SnapshotRegionProcedure ce860946aa8cd08494bb77afea49e9de in 562 msec 2023-10-02T12:31:07,503 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1696249866498_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:31:07,503 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1696249866498_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:31:07,503 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249866498_ns2_test-16962497916551 2023-10-02T12:31:07,504 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249866498_ns2_test-16962497916551 2023-10-02T12:31:07,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-10-02T12:31:07,914 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1696249866498_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:31:07,921 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1696249866498_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:31:07,922 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249866498_ns2_test-16962497916551 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249866498_ns2_test-16962497916551 2023-10-02T12:31:07,923 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1696249866498_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:31:07,923 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249866498_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 }, snapshot procedure id = 91 2023-10-02T12:31:07,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=91, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1696249866498_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } in 1.4010 sec 2023-10-02T12:31:08,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=91 2023-10-02T12:31:08,633 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249868632_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } 2023-10-02T12:31:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249868633 (current time:1696249868633). 2023-10-02T12:31:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:31:08,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249868632_ns3_test-16962497916552 VERSION not specified, setting to 2 2023-10-02T12:31:08,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f4d3111, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:08,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:08,640 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49414, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:08,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:31:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=93, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1696249868632_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } 2023-10-02T12:31:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249868632_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 }, snapshot procedure id = 93 2023-10-02T12:31:08,644 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1696249868632_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:31:08,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-10-02T12:31:08,645 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1696249868632_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:31:08,648 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249868632_ns3_test-16962497916552 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:31:08,650 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1696249868632_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:31:08,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-10-02T12:31:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-10-02T12:31:09,056 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1696249868632_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:31:09,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE; SnapshotRegionProcedure 9d011caaefe2252a8728679a2f592847}] 2023-10-02T12:31:09,059 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=94, ppid=93, state=RUNNABLE; SnapshotRegionProcedure 9d011caaefe2252a8728679a2f592847 2023-10-02T12:31:09,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=94 2023-10-02T12:31:09,210 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. 2023-10-02T12:31:09,212 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.HRegion(2520): Flush status journal for 9d011caaefe2252a8728679a2f592847: 2023-10-02T12:31:09,212 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. for snapshot_1696249868632_ns3_test-16962497916552 completed. 2023-10-02T12:31:09,212 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847.' region-info for snapshot=snapshot_1696249868632_ns3_test-16962497916552 2023-10-02T12:31:09,212 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:31:09,212 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:31:09,217 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. 2023-10-02T12:31:09,217 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=94 2023-10-02T12:31:09,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=94 2023-10-02T12:31:09,218 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249868632_ns3_test-16962497916552 on region 9d011caaefe2252a8728679a2f592847 2023-10-02T12:31:09,219 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=94, ppid=93, state=RUNNABLE; SnapshotRegionProcedure 9d011caaefe2252a8728679a2f592847 2023-10-02T12:31:09,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=94, resume processing ppid=93 2023-10-02T12:31:09,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=94, ppid=93, state=SUCCESS; SnapshotRegionProcedure 9d011caaefe2252a8728679a2f592847 in 163 msec 2023-10-02T12:31:09,221 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1696249868632_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:31:09,222 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1696249868632_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:31:09,223 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1696249868632_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:31:09,223 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249868632_ns3_test-16962497916552 2023-10-02T12:31:09,223 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249868632_ns3_test-16962497916552 2023-10-02T12:31:09,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-10-02T12:31:09,633 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1696249868632_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:31:09,641 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1696249868632_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:31:09,642 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249868632_ns3_test-16962497916552 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249868632_ns3_test-16962497916552 2023-10-02T12:31:09,644 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1696249868632_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:31:09,644 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249868632_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 }, snapshot procedure id = 93 2023-10-02T12:31:09,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=93, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1696249868632_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } in 1.0020 sec 2023-10-02T12:31:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=93 2023-10-02T12:31:09,750 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696249864116 2023-10-02T12:31:09,750 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:31:09,750 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-10-02T12:31:09,750 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249864116_ns2:test-16962497916551 2023-10-02T12:31:09,750 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249866498_ns2_test-16962497916551 to hdfs://localhost:37823/backupUT/backup_1696249864116/ns2/test-16962497916551/ 2023-10-02T12:31:09,750 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:31:09,793 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:31:09,793 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,40619,1696249778965_-87272586_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37823/backupUT/backup_1696249864116/ns2/test-16962497916551, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37823/backupUT/backup_1696249864116/ns2/test-16962497916551/.hbase-snapshot/.tmp/snapshot_1696249866498_ns2_test-16962497916551 2023-10-02T12:31:09,793 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07, snapshotDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249866498_ns2_test-16962497916551. 2023-10-02T12:31:09,803 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249866498_ns2_test-16962497916551 to hdfs://localhost:37823/backupUT/backup_1696249864116/ns2/test-16962497916551/.hbase-snapshot/.tmp/snapshot_1696249866498_ns2_test-16962497916551 2023-10-02T12:31:10,134 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1696249791655 2023-10-02T12:31:10,134 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1696249791655 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:31:10,222 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:10,222 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:10,223 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:10,223 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:11,237 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-2412458334640277527.jar 2023-10-02T12:31:11,238 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:11,239 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:11,239 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:11,240 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:11,240 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:11,241 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:11,241 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:11,242 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:11,242 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:11,243 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:31:11,244 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:31:11,245 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:31:11,245 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:31:11,246 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:31:11,246 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:31:11,247 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:31:11,247 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:31:11,248 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:31:11,248 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:31:11,249 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:31:11,249 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:31:11,250 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:11,251 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:11,252 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:11,252 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:11,253 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:11,253 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:11,254 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:11,304 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249866498_ns2_test-16962497916551' hfile list 2023-10-02T12:31:11,307 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-10-02T12:31:11,409 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249871359/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:31:11,409 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249871359/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:31:11,490 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:31:11,521 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_ output=hdfs://localhost:37823/backupUT/backup_1696249864116/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_ 2023-10-02T12:31:11,521 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 8.321M/sec 2023-10-02T12:31:12,456 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:31:12,458 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:31:12,465 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249866498_ns2_test-16962497916551 2023-10-02T12:31:12,466 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249866498_ns2_test-16962497916551 finished. 2023-10-02T12:31:12,466 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249864116_ns3:test-16962497916552 2023-10-02T12:31:12,466 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249868632_ns3_test-16962497916552 to hdfs://localhost:37823/backupUT/backup_1696249864116/ns3/test-16962497916552/ 2023-10-02T12:31:12,466 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:31:12,501 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:31:12,501 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,40619,1696249778965_-51534802_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37823/backupUT/backup_1696249864116/ns3/test-16962497916552, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37823/backupUT/backup_1696249864116/ns3/test-16962497916552/.hbase-snapshot/.tmp/snapshot_1696249868632_ns3_test-16962497916552 2023-10-02T12:31:12,501 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07, snapshotDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249868632_ns3_test-16962497916552. 2023-10-02T12:31:12,509 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249868632_ns3_test-16962497916552 to hdfs://localhost:37823/backupUT/backup_1696249864116/ns3/test-16962497916552/.hbase-snapshot/.tmp/snapshot_1696249868632_ns3_test-16962497916552 2023-10-02T12:31:12,970 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:12,970 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:12,971 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:12,971 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:14,023 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-5197018244158431067.jar 2023-10-02T12:31:14,023 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:14,024 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:14,024 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:14,024 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:14,025 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:14,025 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:14,025 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:14,026 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:14,026 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:14,027 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:31:14,027 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:31:14,028 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:31:14,028 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:31:14,029 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:31:14,029 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:31:14,029 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:31:14,030 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:31:14,030 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:31:14,030 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:31:14,031 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:31:14,031 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:31:14,032 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:14,032 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:14,033 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:14,033 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:14,034 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:14,034 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:14,035 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:14,092 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249868632_ns3_test-16962497916552' hfile list 2023-10-02T12:31:14,178 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249874136/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:31:14,178 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249874136/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:31:15,212 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:31:15,213 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:31:15,221 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249868632_ns3_test-16962497916552 2023-10-02T12:31:15,221 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249868632_ns3_test-16962497916552 finished. 2023-10-02T12:31:15,221 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1034): ns2:test-16962497916551 2023-10-02T12:31:15,221 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1034): ns3:test-16962497916552 2023-10-02T12:31:15,234 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249864116 2023-10-02T12:31:15,234 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:31:15,641 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37823/backupUT/backup_1696249864116/.backup.manifest 2023-10-02T12:31:15,641 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249864116 2023-10-02T12:31:15,641 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:31:15,648 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37823/backupUT/backup_1696249864116/.backup.manifest 2023-10-02T12:31:15,648 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(358): Backup backup_1696249864116 finished: type=FULL,tablelist=ns2:test-16962497916551;ns3:test-16962497916552,targetRootDir=hdfs://localhost:37823/backupUT,startts=1696249866278,completets=1696249875234,bytescopied=0 2023-10-02T12:31:15,648 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:31:15,648 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249866498_ns2_test-16962497916551 2023-10-02T12:31:15,650 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249866498_ns2_test-16962497916551" 2023-10-02T12:31:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249866498_ns2_test-16962497916551 2023-10-02T12:31:15,654 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249866498_ns2_test-16962497916551 for backup backup_1696249864116 succeeded. 2023-10-02T12:31:15,654 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249868632_ns3_test-16962497916552 2023-10-02T12:31:15,655 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249868632_ns3_test-16962497916552" 2023-10-02T12:31:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249868632_ns3_test-16962497916552 2023-10-02T12:31:15,660 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249868632_ns3_test-16962497916552 for backup backup_1696249864116 succeeded. 2023-10-02T12:31:15,660 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:31:15,667 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:31:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:31:15,671 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:31:15,673 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:31:15,682 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(376): Backup backup_1696249864116 completed. 2023-10-02T12:31:15,682 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client You have to specify either set name or table list to restore Usage: hbase restore [options] backup_path Path to a backup destination root backup_id Backup image ID to restore table(s) Comma-separated list of tables to restore Options: -c Check restore sequence and dependencies only (does not execute the command) -d Enable debug loggings -h,--help Show usage -m A comma separated list of target tables. If specified, each table in must have a mapping -o Overwrite data if any of the restore target tables exists -q Yarn queue name to run backup restore command on -s Backup set to restore, mutually exclusive with -t (table list) -t Table name list, comma-separated. To enable backup, in hbase-site.xml, set: hbase.backup.enable=true hbase.master.logcleaner.plugins=YOUR_PLUGINS,org.apache.hadoop.hbase.backup.master.BackupLogCleaner hbase.procedure.master.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager hbase.procedure.regionserver.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager hbase.coprocessor.region.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.BackupObserver and restart the cluster For more information please see http://hbase.apache.org/book.html#backuprestore 2023-10-02T12:31:15,723 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleDNECommand Thread=597 (was 602), OpenFileDescriptor=682 (was 695), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=605 (was 631), ProcessCount=181 (was 181), AvailableMemoryMB=1963 (was 2045) 2023-10-02T12:31:15,723 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=597 is superior to 500 2023-10-02T12:31:15,747 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultiple Thread=597, OpenFileDescriptor=682, MaxFileDescriptor=60000, SystemLoadAverage=605, ProcessCount=181, AvailableMemoryMB=1963 2023-10-02T12:31:15,747 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=597 is superior to 500 2023-10-02T12:31:15,747 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestFullRestore(120): create full backup image on multiple tables 2023-10-02T12:31:15,754 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@743ed2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:15,787 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:15,789 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54064, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:15,791 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:31:15,792 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45594, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:31:15,807 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:31:15,817 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:31:15,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249875817 (current time:1696249875817). 2023-10-02T12:31:15,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:31:15,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:31:15,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@192b3f6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:15,830 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54076, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:15,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:31:15,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=95, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:31:15,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 95 2023-10-02T12:31:15,835 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:31:15,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-10-02T12:31:15,836 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:31:15,840 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor129.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:31:15,844 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:31:15,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-10-02T12:31:16,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-10-02T12:31:16,253 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:31:16,253 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987}] 2023-10-02T12:31:16,255 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=96, ppid=95, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:31:16,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=96 2023-10-02T12:31:16,407 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:31:16,408 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegion(2819): Flushing 7ec576d712d0f3b0c0592584b88ce987 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-10-02T12:31:16,412 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/b4a8e50177b941698e5089bffe91b251 is 171, key is trslm:hdfs://localhost:37823/backupUT\x00ns2:test-16962497916551/meta:log-roll-map/1696249875224/Put/seqid=0 2023-10-02T12:31:16,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-10-02T12:31:16,817 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/b4a8e50177b941698e5089bffe91b251 2023-10-02T12:31:16,826 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/120da2f382584d43adc4417dfe387fae is 524, key is session:backup_1696249864116/session:context/1696249875671/Put/seqid=0 2023-10-02T12:31:16,833 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/120da2f382584d43adc4417dfe387fae 2023-10-02T12:31:16,839 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/b4a8e50177b941698e5089bffe91b251 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b4a8e50177b941698e5089bffe91b251 2023-10-02T12:31:16,845 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b4a8e50177b941698e5089bffe91b251, entries=6, sequenceid=83, filesize=5.8 K 2023-10-02T12:31:16,846 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/120da2f382584d43adc4417dfe387fae as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/120da2f382584d43adc4417dfe387fae 2023-10-02T12:31:16,852 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/120da2f382584d43adc4417dfe387fae, entries=2, sequenceid=83, filesize=5.5 K 2023-10-02T12:31:16,853 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for 7ec576d712d0f3b0c0592584b88ce987 in 446ms, sequenceid=83, compaction requested=true 2023-10-02T12:31:16,853 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegion(2520): Flush status journal for 7ec576d712d0f3b0c0592584b88ce987: 2023-10-02T12:31:16,853 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. for snapshot_backup_system completed. 2023-10-02T12:31:16,853 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:31:16,853 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:31:16,854 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/85d2568c07cc4be686b8e706b89254ce, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/9bf0c2ed3e5c43ba8bb2287a7e2731fa, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b4a8e50177b941698e5089bffe91b251] hfiles 2023-10-02T12:31:16,854 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98 for snapshot=snapshot_backup_system 2023-10-02T12:31:16,854 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255 for snapshot=snapshot_backup_system 2023-10-02T12:31:16,854 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8 for snapshot=snapshot_backup_system 2023-10-02T12:31:16,854 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/85d2568c07cc4be686b8e706b89254ce for snapshot=snapshot_backup_system 2023-10-02T12:31:16,854 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5 for snapshot=snapshot_backup_system 2023-10-02T12:31:16,854 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/9bf0c2ed3e5c43ba8bb2287a7e2731fa for snapshot=snapshot_backup_system 2023-10-02T12:31:16,854 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b4a8e50177b941698e5089bffe91b251 for snapshot=snapshot_backup_system 2023-10-02T12:31:16,854 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/f8db732eb5cd417eaa236f25fd437245, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/fe4c59a0e2584ff5980ec23dd90e1c0b, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/2129c391bd7447f0b05fea1084f2d47d, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/120da2f382584d43adc4417dfe387fae] hfiles 2023-10-02T12:31:16,854 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (1/8): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0 for snapshot=snapshot_backup_system 2023-10-02T12:31:16,854 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (2/8): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c for snapshot=snapshot_backup_system 2023-10-02T12:31:16,854 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (3/8): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229 for snapshot=snapshot_backup_system 2023-10-02T12:31:16,854 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (4/8): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423 for snapshot=snapshot_backup_system 2023-10-02T12:31:16,854 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (5/8): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/f8db732eb5cd417eaa236f25fd437245 for snapshot=snapshot_backup_system 2023-10-02T12:31:16,854 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (6/8): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/fe4c59a0e2584ff5980ec23dd90e1c0b for snapshot=snapshot_backup_system 2023-10-02T12:31:16,854 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (7/8): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/2129c391bd7447f0b05fea1084f2d47d for snapshot=snapshot_backup_system 2023-10-02T12:31:16,854 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (8/8): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/120da2f382584d43adc4417dfe387fae for snapshot=snapshot_backup_system 2023-10-02T12:31:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-10-02T12:31:17,268 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:31:17,268 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=96 2023-10-02T12:31:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=96 2023-10-02T12:31:17,269 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:31:17,272 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=96, ppid=95, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:31:17,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=96, resume processing ppid=95 2023-10-02T12:31:17,275 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:31:17,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=96, ppid=95, state=SUCCESS; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 in 1.0200 sec 2023-10-02T12:31:17,276 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:31:17,276 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:31:17,276 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:31:17,277 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:31:17,690 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:31:17,704 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:31:17,704 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:31:17,705 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:31:17,705 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 95 2023-10-02T12:31:17,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=95, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8730 sec 2023-10-02T12:31:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=95 2023-10-02T12:31:17,945 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(115): Backup backup_1696249875754 started at 1696249877945. 2023-10-02T12:31:17,947 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(119): Backup session backup_1696249875754 has been started. 2023-10-02T12:31:17,953 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:31:17,954 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:31:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-10-02T12:31:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:31:17,955 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:31:17,955 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:31:17,956 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:31:17,956 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:31:17,957 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:31:17,963 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:31:17,963 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:31:17,963 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:31:17,967 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:31:17,968 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:31:17,968 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:31:17,968 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:17,968 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:31:17,968 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-10-02T12:31:17,968 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:31:17,968 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:31:17,968 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:31:17,969 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:31:17,970 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:31:17,970 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:31:17,974 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:31:17,977 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:31:17,977 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:17,977 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:31:17,977 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:31:17,977 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:31:17,978 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:31:17,978 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:31:17,979 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:17,982 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:17,982 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:31:17,983 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:31:17,983 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@3925250d[Count = 0] remaining members to acquire global barrier 2023-10-02T12:31:17,983 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:31:17,983 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:31:17,988 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:31:17,988 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:31:17,988 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:31:17,988 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:31:17,988 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:31:17,988 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:17,988 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696249866300 highest: 1696249866300 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:17,988 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114:(num 1696249866300) roll requested 2023-10-02T12:31:18,007 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249877989, exclude list is [], retry=0 2023-10-02T12:31:18,011 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:31:18,015 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249866300 with entries=10, filesize=2.67 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249877989 2023-10-02T12:31:18,015 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:31:18,015 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249866300 is not closed yet, will try archiving it next time 2023-10-02T12:31:18,015 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta:.meta(num 1696249866318) roll requested 2023-10-02T12:31:18,033 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249878015.meta, exclude list is [], retry=0 2023-10-02T12:31:18,040 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:31:18,049 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-10-02T12:31:18,049 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696249877989 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:18,050 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249866318.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249878015.meta 2023-10-02T12:31:18,050 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:31:18,051 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249866318.meta is not closed yet, will try archiving it next time 2023-10-02T12:31:18,065 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249866318.meta to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/oldWALs/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249866318.meta 2023-10-02T12:31:18,076 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:31:18,077 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,38255,1696249782114' in zk 2023-10-02T12:31:18,083 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:31:18,083 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:31:18,083 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:18,083 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:31:18,083 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:31:18,083 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:31:18,084 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:31:18,084 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:31:18,085 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:18,085 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:18,085 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:31:18,086 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:18,086 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:18,087 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,38255,1696249782114': 2023-10-02T12:31:18,087 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:31:18,087 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:31:18,087 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:31:18,087 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:31:18,087 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:31:18,087 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:31:18,091 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:31:18,091 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:31:18,091 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:31:18,091 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:31:18,091 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:31:18,091 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:31:18,092 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:31:18,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:31:18,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:18,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:31:18,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:31:18,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:18,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:18,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:31:18,094 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:18,106 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:18,112 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:31:18,112 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:31:18,112 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:31:18,112 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:31:18,112 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:31:18,113 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:31:18,113 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:31:18,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:31:18,218 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249878217_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } 2023-10-02T12:31:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249878218 (current time:1696249878218). 2023-10-02T12:31:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:31:18,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249878217_ns2_test-16962497916551 VERSION not specified, setting to 2 2023-10-02T12:31:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25c91a78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:18,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:18,231 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54090, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:31:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=97, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1696249878217_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } 2023-10-02T12:31:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249878217_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 }, snapshot procedure id = 97 2023-10-02T12:31:18,236 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1696249878217_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:31:18,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=97 2023-10-02T12:31:18,236 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1696249878217_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:31:18,240 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249878217_ns2_test-16962497916551 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor129.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:31:18,243 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1696249878217_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:31:18,248 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1696249878217_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:31:18,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE; SnapshotRegionProcedure ce860946aa8cd08494bb77afea49e9de}] 2023-10-02T12:31:18,249 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=98, ppid=97, state=RUNNABLE; SnapshotRegionProcedure ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:31:18,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=97 2023-10-02T12:31:18,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=98 2023-10-02T12:31:18,401 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. 2023-10-02T12:31:18,401 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.HRegion(2520): Flush status journal for ce860946aa8cd08494bb77afea49e9de: 2023-10-02T12:31:18,401 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. for snapshot_1696249878217_ns2_test-16962497916551 completed. 2023-10-02T12:31:18,401 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de.' region-info for snapshot=snapshot_1696249878217_ns2_test-16962497916551 2023-10-02T12:31:18,402 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:31:18,402 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_] hfiles 2023-10-02T12:31:18,402 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_ for snapshot=snapshot_1696249878217_ns2_test-16962497916551 2023-10-02T12:31:18,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=97 2023-10-02T12:31:18,824 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. 2023-10-02T12:31:18,824 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=98 2023-10-02T12:31:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=98 2023-10-02T12:31:18,825 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249878217_ns2_test-16962497916551 on region ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:31:18,827 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=98, ppid=97, state=RUNNABLE; SnapshotRegionProcedure ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:31:18,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=98, resume processing ppid=97 2023-10-02T12:31:18,830 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1696249878217_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:31:18,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=98, ppid=97, state=SUCCESS; SnapshotRegionProcedure ce860946aa8cd08494bb77afea49e9de in 580 msec 2023-10-02T12:31:18,831 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1696249878217_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:31:18,831 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1696249878217_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:31:18,831 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249878217_ns2_test-16962497916551 2023-10-02T12:31:18,832 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249878217_ns2_test-16962497916551 2023-10-02T12:31:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=97 2023-10-02T12:31:19,255 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1696249878217_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:31:19,268 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1696249878217_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:31:19,269 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249878217_ns2_test-16962497916551 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249878217_ns2_test-16962497916551 2023-10-02T12:31:19,270 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1696249878217_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:31:19,271 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249878217_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 }, snapshot procedure id = 97 2023-10-02T12:31:19,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=97, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1696249878217_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } in 1.0380 sec 2023-10-02T12:31:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=97 2023-10-02T12:31:19,343 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249879343_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } 2023-10-02T12:31:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249879343 (current time:1696249879343). 2023-10-02T12:31:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:31:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249879343_ns3_test-16962497916552 VERSION not specified, setting to 2 2023-10-02T12:31:19,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19dc7f87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:19,358 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:54102, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:31:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=99, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1696249879343_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } 2023-10-02T12:31:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249879343_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 }, snapshot procedure id = 99 2023-10-02T12:31:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-10-02T12:31:19,364 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1696249879343_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:31:19,365 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1696249879343_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:31:19,368 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249879343_ns3_test-16962497916552 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor129.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:31:19,372 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1696249879343_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:31:19,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-10-02T12:31:19,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-10-02T12:31:19,781 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1696249879343_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:31:19,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE; SnapshotRegionProcedure 9d011caaefe2252a8728679a2f592847}] 2023-10-02T12:31:19,784 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=100, ppid=99, state=RUNNABLE; SnapshotRegionProcedure 9d011caaefe2252a8728679a2f592847 2023-10-02T12:31:19,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=100 2023-10-02T12:31:19,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. 2023-10-02T12:31:19,936 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.HRegion(2520): Flush status journal for 9d011caaefe2252a8728679a2f592847: 2023-10-02T12:31:19,936 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. for snapshot_1696249879343_ns3_test-16962497916552 completed. 2023-10-02T12:31:19,936 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847.' region-info for snapshot=snapshot_1696249879343_ns3_test-16962497916552 2023-10-02T12:31:19,936 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:31:19,936 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:31:19,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-10-02T12:31:20,342 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. 2023-10-02T12:31:20,342 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=100 2023-10-02T12:31:20,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=100 2023-10-02T12:31:20,343 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249879343_ns3_test-16962497916552 on region 9d011caaefe2252a8728679a2f592847 2023-10-02T12:31:20,344 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=100, ppid=99, state=RUNNABLE; SnapshotRegionProcedure 9d011caaefe2252a8728679a2f592847 2023-10-02T12:31:20,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=100, resume processing ppid=99 2023-10-02T12:31:20,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=100, ppid=99, state=SUCCESS; SnapshotRegionProcedure 9d011caaefe2252a8728679a2f592847 in 563 msec 2023-10-02T12:31:20,347 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1696249879343_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:31:20,347 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1696249879343_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:31:20,348 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1696249879343_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:31:20,348 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249879343_ns3_test-16962497916552 2023-10-02T12:31:20,349 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249879343_ns3_test-16962497916552 2023-10-02T12:31:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-10-02T12:31:20,759 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1696249879343_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:31:20,763 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1696249879343_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:31:20,764 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249879343_ns3_test-16962497916552 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249879343_ns3_test-16962497916552 2023-10-02T12:31:20,765 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1696249879343_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:31:20,765 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249879343_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 }, snapshot procedure id = 99 2023-10-02T12:31:20,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=99, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1696249879343_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } in 1.4050 sec 2023-10-02T12:31:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=99 2023-10-02T12:31:21,472 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696249875754 2023-10-02T12:31:21,472 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:31:21,473 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-10-02T12:31:21,473 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249875754_ns2:test-16962497916551 2023-10-02T12:31:21,473 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249878217_ns2_test-16962497916551 to hdfs://localhost:37823/backupUT/backup_1696249875754/ns2/test-16962497916551/ 2023-10-02T12:31:21,473 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:31:21,496 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:31:21,496 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,40619,1696249778965_-1569394165_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37823/backupUT/backup_1696249875754/ns2/test-16962497916551, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37823/backupUT/backup_1696249875754/ns2/test-16962497916551/.hbase-snapshot/.tmp/snapshot_1696249878217_ns2_test-16962497916551 2023-10-02T12:31:21,496 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07, snapshotDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249878217_ns2_test-16962497916551. 2023-10-02T12:31:21,504 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249878217_ns2_test-16962497916551 to hdfs://localhost:37823/backupUT/backup_1696249875754/ns2/test-16962497916551/.hbase-snapshot/.tmp/snapshot_1696249878217_ns2_test-16962497916551 2023-10-02T12:31:21,522 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:21,523 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:21,524 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:21,524 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:22,532 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-997535410471960718.jar 2023-10-02T12:31:22,532 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:22,533 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:22,533 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:22,534 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:22,534 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:22,534 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:22,535 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:22,535 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:22,535 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:22,536 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:31:22,536 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:31:22,537 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:31:22,537 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:31:22,538 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:31:22,538 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:31:22,538 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:31:22,539 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:31:22,539 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:31:22,539 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:31:22,540 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:31:22,541 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:31:22,542 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:22,542 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:22,543 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:22,544 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:22,544 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:22,545 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:22,546 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:22,593 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249878217_ns2_test-16962497916551' hfile list 2023-10-02T12:31:22,596 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-10-02T12:31:22,739 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249882644/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:31:22,739 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249882644/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:31:22,801 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:31:22,828 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_ output=hdfs://localhost:37823/backupUT/backup_1696249875754/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_ 2023-10-02T12:31:22,828 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 4.160M/sec 2023-10-02T12:31:23,777 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:31:23,778 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:31:23,785 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249878217_ns2_test-16962497916551 2023-10-02T12:31:23,786 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249878217_ns2_test-16962497916551 finished. 2023-10-02T12:31:23,786 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249875754_ns3:test-16962497916552 2023-10-02T12:31:23,786 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249879343_ns3_test-16962497916552 to hdfs://localhost:37823/backupUT/backup_1696249875754/ns3/test-16962497916552/ 2023-10-02T12:31:23,786 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:31:23,809 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:31:23,809 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,40619,1696249778965_-429746987_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37823/backupUT/backup_1696249875754/ns3/test-16962497916552, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37823/backupUT/backup_1696249875754/ns3/test-16962497916552/.hbase-snapshot/.tmp/snapshot_1696249879343_ns3_test-16962497916552 2023-10-02T12:31:23,809 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07, snapshotDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249879343_ns3_test-16962497916552. 2023-10-02T12:31:23,815 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249879343_ns3_test-16962497916552 to hdfs://localhost:37823/backupUT/backup_1696249875754/ns3/test-16962497916552/.hbase-snapshot/.tmp/snapshot_1696249879343_ns3_test-16962497916552 2023-10-02T12:31:24,234 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:24,235 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:24,236 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:24,236 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:25,257 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-5514745842859582386.jar 2023-10-02T12:31:25,258 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:25,258 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:25,259 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:25,259 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:25,260 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:25,260 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:25,261 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:25,261 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:25,261 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:25,262 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:31:25,263 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:31:25,264 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:31:25,264 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:31:25,265 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:31:25,265 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:31:25,266 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:31:25,266 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:31:25,267 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:31:25,267 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:31:25,268 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:31:25,268 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:31:25,269 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:25,269 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:25,270 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:25,271 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:25,271 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:25,272 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:25,273 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:25,326 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249879343_ns3_test-16962497916552' hfile list 2023-10-02T12:31:25,419 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249885375/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:31:25,419 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249885375/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:31:26,453 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:31:26,455 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:31:26,461 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249879343_ns3_test-16962497916552 2023-10-02T12:31:26,461 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249879343_ns3_test-16962497916552 finished. 2023-10-02T12:31:26,461 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1034): ns2:test-16962497916551 2023-10-02T12:31:26,461 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1034): ns3:test-16962497916552 2023-10-02T12:31:26,468 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249875754 2023-10-02T12:31:26,468 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:31:26,475 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37823/backupUT/backup_1696249875754/.backup.manifest 2023-10-02T12:31:26,475 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249875754 2023-10-02T12:31:26,475 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:31:26,483 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37823/backupUT/backup_1696249875754/.backup.manifest 2023-10-02T12:31:26,483 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(358): Backup backup_1696249875754 finished: type=FULL,tablelist=ns2:test-16962497916551;ns3:test-16962497916552,targetRootDir=hdfs://localhost:37823/backupUT,startts=1696249877945,completets=1696249886468,bytescopied=0 2023-10-02T12:31:26,483 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:31:26,483 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249878217_ns2_test-16962497916551 2023-10-02T12:31:26,485 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249878217_ns2_test-16962497916551" 2023-10-02T12:31:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249878217_ns2_test-16962497916551 2023-10-02T12:31:26,492 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249878217_ns2_test-16962497916551 for backup backup_1696249875754 succeeded. 2023-10-02T12:31:26,492 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249879343_ns3_test-16962497916552 2023-10-02T12:31:26,494 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249879343_ns3_test-16962497916552" 2023-10-02T12:31:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249879343_ns3_test-16962497916552 2023-10-02T12:31:26,498 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249879343_ns3_test-16962497916552 for backup backup_1696249875754 succeeded. 2023-10-02T12:31:26,499 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:31:26,505 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:31:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:31:26,512 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:31:26,514 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:31:26,521 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(376): Backup backup_1696249875754 completed. 2023-10-02T12:31:26,521 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:26,541 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(101): HBase table ns2:table2 does not exist. It will be created during restore process 2023-10-02T12:31:26,542 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(101): HBase table ns3:table3_restore does not exist. It will be created during restore process 2023-10-02T12:31:26,543 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249875754/.backup.manifest 2023-10-02T12:31:26,545 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249875754/.backup.manifest 2023-10-02T12:31:26,546 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249875754/.backup.manifest 2023-10-02T12:31:26,548 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249875754/.backup.manifest 2023-10-02T12:31:26,549 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249875754/.backup.manifest 2023-10-02T12:31:26,551 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249875754/.backup.manifest 2023-10-02T12:31:26,551 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-16962497916551' to 'ns2:table2' from full backup image hdfs://localhost:37823/backupUT/backup_1696249875754/ns2/test-16962497916551 2023-10-02T12:31:26,562 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(487): Creating target table 'ns2:table2' 2023-10-02T12:31:26,562 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:37823/backupUT/backup_1696249875754/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:31:26,563 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:37823/backupUT/backup_1696249875754/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f in region [hdfs://localhost:37823/backupUT/backup_1696249875754/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de] 2023-10-02T12:31:26,567 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:37823/backupUT/backup_1696249875754/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_ first=row0 last=row98 2023-10-02T12:31:26,569 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:31:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=101, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:table2 2023-10-02T12:31:26,572 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:31:26,572 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "table2" procId is: 101 2023-10-02T12:31:26,573 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:31:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=101 2023-10-02T12:31:26,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=101 2023-10-02T12:31:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=101 2023-10-02T12:31:26,999 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => f478c7e689645536339b5d7f416ec426, NAME => 'ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:31:27,008 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:31:27,008 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1668): Closing f478c7e689645536339b5d7f416ec426, disabling compactions & flushes 2023-10-02T12:31:27,008 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426. 2023-10-02T12:31:27,008 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426. 2023-10-02T12:31:27,008 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426. after waiting 0 ms 2023-10-02T12:31:27,008 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426. 2023-10-02T12:31:27,008 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1909): Closed ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426. 2023-10-02T12:31:27,008 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1622): Region close journal for f478c7e689645536339b5d7f416ec426: 2023-10-02T12:31:27,009 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:31:27,009 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1696249887009"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249887009"}]},"ts":"1696249887009"} 2023-10-02T12:31:27,011 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:31:27,012 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:31:27,012 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249887012"}]},"ts":"1696249887012"} 2023-10-02T12:31:27,013 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2023-10-02T12:31:27,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=f478c7e689645536339b5d7f416ec426, ASSIGN}] 2023-10-02T12:31:27,019 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=f478c7e689645536339b5d7f416ec426, ASSIGN 2023-10-02T12:31:27,020 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:table2, region=f478c7e689645536339b5d7f416ec426, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38255,1696249782114; forceNewPlan=false, retain=false 2023-10-02T12:31:27,171 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=102 updating hbase:meta row=f478c7e689645536339b5d7f416ec426, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:27,173 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE; OpenRegionProcedure f478c7e689645536339b5d7f416ec426, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:31:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=101 2023-10-02T12:31:27,329 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] handler.AssignRegionHandler(130): Open ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426. 2023-10-02T12:31:27,330 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7254): Opening region: {ENCODED => f478c7e689645536339b5d7f416ec426, NAME => 'ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:31:27,330 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:31:27,330 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 f478c7e689645536339b5d7f416ec426 2023-10-02T12:31:27,330 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(888): Instantiated ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:31:27,330 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7296): checking encryption for f478c7e689645536339b5d7f416ec426 2023-10-02T12:31:27,330 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7299): checking classloading for f478c7e689645536339b5d7f416ec426 2023-10-02T12:31:27,332 INFO [StoreOpener-f478c7e689645536339b5d7f416ec426-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region f478c7e689645536339b5d7f416ec426 2023-10-02T12:31:27,334 INFO [StoreOpener-f478c7e689645536339b5d7f416ec426-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f478c7e689645536339b5d7f416ec426 columnFamilyName f 2023-10-02T12:31:27,334 DEBUG [StoreOpener-f478c7e689645536339b5d7f416ec426-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:31:27,335 INFO [StoreOpener-f478c7e689645536339b5d7f416ec426-1 {}] regionserver.HStore(324): Store=f478c7e689645536339b5d7f416ec426/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:31:27,336 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/f478c7e689645536339b5d7f416ec426 2023-10-02T12:31:27,336 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/f478c7e689645536339b5d7f416ec426 2023-10-02T12:31:27,340 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1076): writing seq id for f478c7e689645536339b5d7f416ec426 2023-10-02T12:31:27,342 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/f478c7e689645536339b5d7f416ec426/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:31:27,343 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1093): Opened f478c7e689645536339b5d7f416ec426; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9874485920, jitterRate=-0.0803668349981308}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:31:27,344 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(992): Region open journal for f478c7e689645536339b5d7f416ec426: 2023-10-02T12:31:27,345 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426., pid=103, masterSystemTime=1696249887325 2023-10-02T12:31:27,347 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=102 updating hbase:meta row=f478c7e689645536339b5d7f416ec426, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:27,348 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426. 2023-10-02T12:31:27,348 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] handler.AssignRegionHandler(158): Opened ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426. 2023-10-02T12:31:27,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=103, resume processing ppid=102 2023-10-02T12:31:27,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=103, ppid=102, state=SUCCESS; OpenRegionProcedure f478c7e689645536339b5d7f416ec426, server=jenkins-hbase3.apache.org,38255,1696249782114 in 178 msec 2023-10-02T12:31:27,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=102, resume processing ppid=101 2023-10-02T12:31:27,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=102, ppid=101, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=f478c7e689645536339b5d7f416ec426, ASSIGN in 336 msec 2023-10-02T12:31:27,356 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:31:27,356 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249887356"}]},"ts":"1696249887356"} 2023-10-02T12:31:27,357 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2023-10-02T12:31:27,360 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:31:27,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=101, state=SUCCESS; CreateTableProcedure table=ns2:table2 in 791 msec 2023-10-02T12:31:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=101 2023-10-02T12:31:27,682 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:37823/backupUT/backup_1696249875754/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de from hbase tables ns2:test-16962497916551 to tables ns2:table2 2023-10-02T12:31:27,682 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-16962497916551 into ns2:table2 2023-10-02T12:31:27,684 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-table2-1696249887682 from hdfs://localhost:37823/backupUT/backup_1696249875754/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:31:27,722 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e0180fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:27,723 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:31:27,724 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37558, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:31:27,725 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-10-02T12:31:27,725 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:table2 2023-10-02T12:31:27,729 DEBUG [hconnection-0x5860b805-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:27,730 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45018, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:27,732 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:table2: 2023-10-02T12:31:27,732 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-10-02T12:31:27,732 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/partitions_42bbc7b7-5a79-464d-84b7-55c633889e4c 2023-10-02T12:31:28,138 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:28,139 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:28,139 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:28,139 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:29,146 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-6329864884688719035.jar 2023-10-02T12:31:29,146 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:29,147 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:29,147 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:29,148 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:29,148 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:29,148 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:29,149 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:29,149 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:29,149 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:29,150 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:31:29,151 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:31:29,151 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:31:29,151 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:31:29,152 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:31:29,152 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:31:29,153 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:31:29,153 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:31:29,154 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:31:29,155 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:31:29,155 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:31:29,156 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:31:29,157 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:29,157 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:29,158 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:29,158 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:29,159 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:29,159 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:29,160 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:29,161 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:table2 2023-10-02T12:31:29,161 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:29,161 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-10-02T12:31:29,161 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-10-02T12:31:29,162 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:31:29,175 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-10-02T12:31:29,271 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249889227/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:31:29,271 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249889227/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:31:29,320 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:37823/backupUT/backup_1696249875754/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_ 2023-10-02T12:31:29,324 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-10-02T12:31:29,347 DEBUG [pool-396-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d2135e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:29,348 DEBUG [hconnection-0x70c15c4e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:29,349 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:45022, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:29,351 DEBUG [pool-396-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:29,352 DEBUG [pool-396-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-10-02T12:31:29,352 DEBUG [pool-396-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-10-02T12:31:29,358 DEBUG [pool-396-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-table2-1696249887682/_temporary/0/_temporary/attempt_local1913812159_0016_r_000000_0/f/f692baf7d82440c6bf39aadfa48b9011 is 37, key is row10/f:q1/1696249795545/Put/seqid=0 2023-10-02T12:31:29,359 WARN [Thread-1713 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:38255] Specified favored nodes: [jenkins-hbase3.apache.org:38255] 2023-10-02T12:31:30,134 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2023-10-02T12:31:30,134 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_table2 Metrics about Tables on a single HBase RegionServer 2023-10-02T12:31:30,316 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-table2-1696249887682 2023-10-02T12:31:30,322 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14182588, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:30,323 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:30,324 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:56826, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:30,329 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-table2-1696249887682/_SUCCESS 2023-10-02T12:31:30,333 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:31:30,334 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47326, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:31:30,342 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-table2-1696249887682/f/f692baf7d82440c6bf39aadfa48b9011 first=Optional[row0] last=Optional[row98] 2023-10-02T12:31:30,343 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426., hostname=jenkins-hbase3.apache.org,38255,1696249782114, seqNum=2 for row with hfile group [{f,hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-table2-1696249887682/f/f692baf7d82440c6bf39aadfa48b9011}] 2023-10-02T12:31:30,349 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-table2-1696249887682/f/f692baf7d82440c6bf39aadfa48b9011 for inclusion in f478c7e689645536339b5d7f416ec426/f 2023-10-02T12:31:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-10-02T12:31:30,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(625): Region bounds: first= last= 2023-10-02T12:31:30,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HRegion(2520): Flush status journal for f478c7e689645536339b5d7f416ec426: 2023-10-02T12:31:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-table2-1696249887682/f/f692baf7d82440c6bf39aadfa48b9011 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__ns2_table2__mcqogbuojil6gm9kk2tosn7vrl4dnpqc7qptl07jjrd3943i3ggsm7cfrs0j5h61/f/f692baf7d82440c6bf39aadfa48b9011 2023-10-02T12:31:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__ns2_table2__mcqogbuojil6gm9kk2tosn7vrl4dnpqc7qptl07jjrd3943i3ggsm7cfrs0j5h61/f/f692baf7d82440c6bf39aadfa48b9011 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/f478c7e689645536339b5d7f416ec426/f/ac5c1e9baabf4b1a86d44cef2f9a039d_SeqId_4_ 2023-10-02T12:31:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c574429, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:30,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:31:30,364 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47330, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:31:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:30,369 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:56840, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:30,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:30,387 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__ns2_table2__mcqogbuojil6gm9kk2tosn7vrl4dnpqc7qptl07jjrd3943i3ggsm7cfrs0j5h61/f/f692baf7d82440c6bf39aadfa48b9011 into f478c7e689645536339b5d7f416ec426/f as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/f478c7e689645536339b5d7f416ec426/f/ac5c1e9baabf4b1a86d44cef2f9a039d_SeqId_4_ - updating store file list. 2023-10-02T12:31:30,392 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/f478c7e689645536339b5d7f416ec426/f/ac5c1e9baabf4b1a86d44cef2f9a039d_SeqId_4_ into f478c7e689645536339b5d7f416ec426/f 2023-10-02T12:31:30,392 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__ns2_table2__mcqogbuojil6gm9kk2tosn7vrl4dnpqc7qptl07jjrd3943i3ggsm7cfrs0j5h61/f/f692baf7d82440c6bf39aadfa48b9011 into f478c7e689645536339b5d7f416ec426/f (new location: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/f478c7e689645536339b5d7f416ec426/f/ac5c1e9baabf4b1a86d44cef2f9a039d_SeqId_4_) 2023-10-02T12:31:30,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__ns2_table2__mcqogbuojil6gm9kk2tosn7vrl4dnpqc7qptl07jjrd3943i3ggsm7cfrs0j5h61/f/f692baf7d82440c6bf39aadfa48b9011 2023-10-02T12:31:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e2e6989, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:31:30,400 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47342, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-10-02T12:31:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:30,407 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:56852, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-10-02T12:31:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:30,426 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:30,426 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-10-02T12:31:30,426 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-10-02T12:31:30,426 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(233): Backup: backup_1696249875754 hdfs://localhost:37823/backupUT/backup_1696249875754/ns2/test-16962497916551/ 2023-10-02T12:31:30,427 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249875754/.backup.manifest 2023-10-02T12:31:30,430 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249875754/.backup.manifest 2023-10-02T12:31:30,430 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-16962497916552' to 'ns3:table3_restore' from full backup image hdfs://localhost:37823/backupUT/backup_1696249875754/ns3/test-16962497916552 2023-10-02T12:31:30,436 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:37823/backupUT/backup_1696249875754/ns3/test-16962497916552/archive/data/ns3/test-16962497916552 does not exists 2023-10-02T12:31:30,436 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-16962497916552, will only create table 2023-10-02T12:31:30,437 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(487): Creating target table 'ns3:table3_restore' 2023-10-02T12:31:30,439 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:31:30,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:table3_restore 2023-10-02T12:31:30,442 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:31:30,442 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "table3_restore" procId is: 104 2023-10-02T12:31:30,443 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:31:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=104 2023-10-02T12:31:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=104 2023-10-02T12:31:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=104 2023-10-02T12:31:30,853 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => f19c6990ee00a4ba87be3777476b5d5b, NAME => 'ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:31:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=104 2023-10-02T12:31:31,260 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:31:31,260 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1668): Closing f19c6990ee00a4ba87be3777476b5d5b, disabling compactions & flushes 2023-10-02T12:31:31,260 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b. 2023-10-02T12:31:31,260 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b. 2023-10-02T12:31:31,260 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b. after waiting 0 ms 2023-10-02T12:31:31,260 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b. 2023-10-02T12:31:31,260 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b. 2023-10-02T12:31:31,260 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1622): Region close journal for f19c6990ee00a4ba87be3777476b5d5b: 2023-10-02T12:31:31,261 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:31:31,262 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1696249891262"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249891262"}]},"ts":"1696249891262"} 2023-10-02T12:31:31,263 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:31:31,264 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:31:31,264 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249891264"}]},"ts":"1696249891264"} 2023-10-02T12:31:31,265 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLING in hbase:meta 2023-10-02T12:31:31,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=f19c6990ee00a4ba87be3777476b5d5b, ASSIGN}] 2023-10-02T12:31:31,275 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=f19c6990ee00a4ba87be3777476b5d5b, ASSIGN 2023-10-02T12:31:31,276 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:table3_restore, region=f19c6990ee00a4ba87be3777476b5d5b, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38255,1696249782114; forceNewPlan=false, retain=false 2023-10-02T12:31:31,428 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=105 updating hbase:meta row=f19c6990ee00a4ba87be3777476b5d5b, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:31,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE; OpenRegionProcedure f19c6990ee00a4ba87be3777476b5d5b, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:31:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=104 2023-10-02T12:31:31,585 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(130): Open ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b. 2023-10-02T12:31:31,585 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7254): Opening region: {ENCODED => f19c6990ee00a4ba87be3777476b5d5b, NAME => 'ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:31:31,586 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:31:31,586 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table3_restore f19c6990ee00a4ba87be3777476b5d5b 2023-10-02T12:31:31,586 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:31:31,586 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7296): checking encryption for f19c6990ee00a4ba87be3777476b5d5b 2023-10-02T12:31:31,586 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7299): checking classloading for f19c6990ee00a4ba87be3777476b5d5b 2023-10-02T12:31:31,589 INFO [StoreOpener-f19c6990ee00a4ba87be3777476b5d5b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region f19c6990ee00a4ba87be3777476b5d5b 2023-10-02T12:31:31,590 INFO [StoreOpener-f19c6990ee00a4ba87be3777476b5d5b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f19c6990ee00a4ba87be3777476b5d5b columnFamilyName f 2023-10-02T12:31:31,590 DEBUG [StoreOpener-f19c6990ee00a4ba87be3777476b5d5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:31:31,591 INFO [StoreOpener-f19c6990ee00a4ba87be3777476b5d5b-1 {}] regionserver.HStore(324): Store=f19c6990ee00a4ba87be3777476b5d5b/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:31:31,592 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/table3_restore/f19c6990ee00a4ba87be3777476b5d5b 2023-10-02T12:31:31,592 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/table3_restore/f19c6990ee00a4ba87be3777476b5d5b 2023-10-02T12:31:31,594 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1076): writing seq id for f19c6990ee00a4ba87be3777476b5d5b 2023-10-02T12:31:31,597 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/table3_restore/f19c6990ee00a4ba87be3777476b5d5b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:31:31,597 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1093): Opened f19c6990ee00a4ba87be3777476b5d5b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9459952480, jitterRate=-0.1189732700586319}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:31:31,598 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(992): Region open journal for f19c6990ee00a4ba87be3777476b5d5b: 2023-10-02T12:31:31,599 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b., pid=106, masterSystemTime=1696249891582 2023-10-02T12:31:31,600 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b. 2023-10-02T12:31:31,600 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(158): Opened ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b. 2023-10-02T12:31:31,601 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=105 updating hbase:meta row=f19c6990ee00a4ba87be3777476b5d5b, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:31,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=106, resume processing ppid=105 2023-10-02T12:31:31,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=106, ppid=105, state=SUCCESS; OpenRegionProcedure f19c6990ee00a4ba87be3777476b5d5b, server=jenkins-hbase3.apache.org,38255,1696249782114 in 172 msec 2023-10-02T12:31:31,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=105, resume processing ppid=104 2023-10-02T12:31:31,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=105, ppid=104, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=f19c6990ee00a4ba87be3777476b5d5b, ASSIGN in 334 msec 2023-10-02T12:31:31,607 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:31:31,607 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249891607"}]},"ts":"1696249891607"} 2023-10-02T12:31:31,610 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLED in hbase:meta 2023-10-02T12:31:31,613 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:31:31,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=104, state=SUCCESS; CreateTableProcedure table=ns3:table3_restore in 1.1740 sec 2023-10-02T12:31:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=104 2023-10-02T12:31:32,555 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-10-02T12:31:32,555 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(233): Backup: backup_1696249875754 hdfs://localhost:37823/backupUT/backup_1696249875754/ns3/test-16962497916552/ 2023-10-02T12:31:32,555 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-10-02T12:31:32,557 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable ns2:table2 2023-10-02T12:31:32,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:table2 2023-10-02T12:31:32,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=107 2023-10-02T12:31:32,562 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249892562"}]},"ts":"1696249892562"} 2023-10-02T12:31:32,564 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLING in hbase:meta 2023-10-02T12:31:32,566 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set ns2:table2 to state=DISABLING 2023-10-02T12:31:32,567 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=f478c7e689645536339b5d7f416ec426, UNASSIGN}] 2023-10-02T12:31:32,568 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=108, ppid=107, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=f478c7e689645536339b5d7f416ec426, UNASSIGN 2023-10-02T12:31:32,570 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=108 updating hbase:meta row=f478c7e689645536339b5d7f416ec426, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:32,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE; CloseRegionProcedure f478c7e689645536339b5d7f416ec426, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:31:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=107 2023-10-02T12:31:32,724 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] handler.UnassignRegionHandler(119): Close f478c7e689645536339b5d7f416ec426 2023-10-02T12:31:32,724 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1668): Closing f478c7e689645536339b5d7f416ec426, disabling compactions & flushes 2023-10-02T12:31:32,725 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1690): Closing region ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426. 2023-10-02T12:31:32,725 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426. 2023-10-02T12:31:32,725 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426. after waiting 0 ms 2023-10-02T12:31:32,725 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426. 2023-10-02T12:31:32,735 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/f478c7e689645536339b5d7f416ec426/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-10-02T12:31:32,736 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:31:32,737 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1909): Closed ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426. 2023-10-02T12:31:32,737 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1622): Region close journal for f478c7e689645536339b5d7f416ec426: 2023-10-02T12:31:32,741 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] handler.UnassignRegionHandler(163): Closed f478c7e689645536339b5d7f416ec426 2023-10-02T12:31:32,743 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=108 updating hbase:meta row=f478c7e689645536339b5d7f416ec426, regionState=CLOSED 2023-10-02T12:31:32,746 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=109, resume processing ppid=108 2023-10-02T12:31:32,746 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=109, ppid=108, state=SUCCESS; CloseRegionProcedure f478c7e689645536339b5d7f416ec426, server=jenkins-hbase3.apache.org,38255,1696249782114 in 173 msec 2023-10-02T12:31:32,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=108, resume processing ppid=107 2023-10-02T12:31:32,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=108, ppid=107, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=f478c7e689645536339b5d7f416ec426, UNASSIGN in 179 msec 2023-10-02T12:31:32,750 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249892750"}]},"ts":"1696249892750"} 2023-10-02T12:31:32,752 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLED in hbase:meta 2023-10-02T12:31:32,754 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set ns2:table2 to state=DISABLED 2023-10-02T12:31:32,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=107, state=SUCCESS; DisableTableProcedure table=ns2:table2 in 196 msec 2023-10-02T12:31:32,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=107 2023-10-02T12:31:32,867 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete ns2:table2 2023-10-02T12:31:32,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=110, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns2:table2 2023-10-02T12:31:32,870 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=110, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-10-02T12:31:32,871 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=110, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns2:table2 2023-10-02T12:31:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=110 2023-10-02T12:31:32,873 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/f478c7e689645536339b5d7f416ec426 2023-10-02T12:31:32,875 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/f478c7e689645536339b5d7f416ec426/f, FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/f478c7e689645536339b5d7f416ec426/recovered.edits] 2023-10-02T12:31:32,883 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/f478c7e689645536339b5d7f416ec426/f/ac5c1e9baabf4b1a86d44cef2f9a039d_SeqId_4_ to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/archive/data/ns2/table2/f478c7e689645536339b5d7f416ec426/f/ac5c1e9baabf4b1a86d44cef2f9a039d_SeqId_4_ 2023-10-02T12:31:32,887 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/f478c7e689645536339b5d7f416ec426/recovered.edits/6.seqid to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/archive/data/ns2/table2/f478c7e689645536339b5d7f416ec426/recovered.edits/6.seqid 2023-10-02T12:31:32,888 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/f478c7e689645536339b5d7f416ec426 2023-10-02T12:31:32,888 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(293): Archived ns2:table2 regions 2023-10-02T12:31:32,888 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:31:32,895 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=110, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns2:table2 2023-10-02T12:31:32,904 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:table2 from hbase:meta 2023-10-02T12:31:32,908 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:table2' descriptor. 2023-10-02T12:31:32,909 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=110, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns2:table2 2023-10-02T12:31:32,909 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:table2' from region states. 2023-10-02T12:31:32,910 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696249892909"}]},"ts":"9223372036854775807"} 2023-10-02T12:31:32,912 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:31:32,912 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => f478c7e689645536339b5d7f416ec426, NAME => 'ns2:table2,,1696249886568.f478c7e689645536339b5d7f416ec426.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:31:32,912 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:table2' as deleted. 2023-10-02T12:31:32,913 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1696249892912"}]},"ts":"9223372036854775807"} 2023-10-02T12:31:32,926 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:table2 state from META 2023-10-02T12:31:32,928 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(127): Finished pid=110, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-10-02T12:31:32,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=110, state=SUCCESS; DeleteTableProcedure table=ns2:table2 in 61 msec 2023-10-02T12:31:32,937 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:table3_restore' 2023-10-02T12:31:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=110 2023-10-02T12:31:32,974 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable ns3:table3_restore 2023-10-02T12:31:32,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:table3_restore 2023-10-02T12:31:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=111 2023-10-02T12:31:32,979 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249892979"}]},"ts":"1696249892979"} 2023-10-02T12:31:32,980 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLING in hbase:meta 2023-10-02T12:31:32,982 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set ns3:table3_restore to state=DISABLING 2023-10-02T12:31:32,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=f19c6990ee00a4ba87be3777476b5d5b, UNASSIGN}] 2023-10-02T12:31:32,984 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=112, ppid=111, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=f19c6990ee00a4ba87be3777476b5d5b, UNASSIGN 2023-10-02T12:31:32,984 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=112 updating hbase:meta row=f19c6990ee00a4ba87be3777476b5d5b, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:32,986 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE; CloseRegionProcedure f19c6990ee00a4ba87be3777476b5d5b, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:31:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=111 2023-10-02T12:31:33,151 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] handler.UnassignRegionHandler(119): Close f19c6990ee00a4ba87be3777476b5d5b 2023-10-02T12:31:33,152 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1668): Closing f19c6990ee00a4ba87be3777476b5d5b, disabling compactions & flushes 2023-10-02T12:31:33,153 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b. 2023-10-02T12:31:33,153 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b. 2023-10-02T12:31:33,153 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b. after waiting 0 ms 2023-10-02T12:31:33,153 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b. 2023-10-02T12:31:33,168 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/table3_restore/f19c6990ee00a4ba87be3777476b5d5b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-10-02T12:31:33,169 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:31:33,170 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b. 2023-10-02T12:31:33,170 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1622): Region close journal for f19c6990ee00a4ba87be3777476b5d5b: 2023-10-02T12:31:33,173 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] handler.UnassignRegionHandler(163): Closed f19c6990ee00a4ba87be3777476b5d5b 2023-10-02T12:31:33,174 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=112 updating hbase:meta row=f19c6990ee00a4ba87be3777476b5d5b, regionState=CLOSED 2023-10-02T12:31:33,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=113, resume processing ppid=112 2023-10-02T12:31:33,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=113, ppid=112, state=SUCCESS; CloseRegionProcedure f19c6990ee00a4ba87be3777476b5d5b, server=jenkins-hbase3.apache.org,38255,1696249782114 in 189 msec 2023-10-02T12:31:33,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=112, resume processing ppid=111 2023-10-02T12:31:33,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=112, ppid=111, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=f19c6990ee00a4ba87be3777476b5d5b, UNASSIGN in 195 msec 2023-10-02T12:31:33,180 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249893180"}]},"ts":"1696249893180"} 2023-10-02T12:31:33,182 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLED in hbase:meta 2023-10-02T12:31:33,190 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set ns3:table3_restore to state=DISABLED 2023-10-02T12:31:33,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=111, state=SUCCESS; DisableTableProcedure table=ns3:table3_restore in 216 msec 2023-10-02T12:31:33,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=111 2023-10-02T12:31:33,283 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete ns3:table3_restore 2023-10-02T12:31:33,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=114, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns3:table3_restore 2023-10-02T12:31:33,287 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=114, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-10-02T12:31:33,288 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=114, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-10-02T12:31:33,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=114 2023-10-02T12:31:33,296 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/table3_restore/f19c6990ee00a4ba87be3777476b5d5b 2023-10-02T12:31:33,302 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/table3_restore/f19c6990ee00a4ba87be3777476b5d5b/f, FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/table3_restore/f19c6990ee00a4ba87be3777476b5d5b/recovered.edits] 2023-10-02T12:31:33,309 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/table3_restore/f19c6990ee00a4ba87be3777476b5d5b/recovered.edits/4.seqid to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/archive/data/ns3/table3_restore/f19c6990ee00a4ba87be3777476b5d5b/recovered.edits/4.seqid 2023-10-02T12:31:33,310 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/table3_restore/f19c6990ee00a4ba87be3777476b5d5b 2023-10-02T12:31:33,310 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived ns3:table3_restore regions 2023-10-02T12:31:33,314 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=114, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-10-02T12:31:33,316 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:table3_restore from hbase:meta 2023-10-02T12:31:33,318 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:table3_restore' descriptor. 2023-10-02T12:31:33,319 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=114, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-10-02T12:31:33,319 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:table3_restore' from region states. 2023-10-02T12:31:33,319 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696249893319"}]},"ts":"9223372036854775807"} 2023-10-02T12:31:33,321 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:31:33,321 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => f19c6990ee00a4ba87be3777476b5d5b, NAME => 'ns3:table3_restore,,1696249890438.f19c6990ee00a4ba87be3777476b5d5b.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:31:33,321 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:table3_restore' as deleted. 2023-10-02T12:31:33,321 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1696249893321"}]},"ts":"9223372036854775807"} 2023-10-02T12:31:33,323 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:table3_restore state from META 2023-10-02T12:31:33,326 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(127): Finished pid=114, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-10-02T12:31:33,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=114, state=SUCCESS; DeleteTableProcedure table=ns3:table3_restore in 43 msec 2023-10-02T12:31:33,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=114 2023-10-02T12:31:33,411 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultiple Thread=600 (was 597) - Thread LEAK? -, OpenFileDescriptor=680 (was 682), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=550 (was 605), ProcessCount=181 (was 181), AvailableMemoryMB=1856 (was 1963) 2023-10-02T12:31:33,411 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=600 is superior to 500 2023-10-02T12:31:33,433 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleOverwriteCommand Thread=600, OpenFileDescriptor=680, MaxFileDescriptor=60000, SystemLoadAverage=550, ProcessCount=181, AvailableMemoryMB=1855 2023-10-02T12:31:33,433 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=600 is superior to 500 2023-10-02T12:31:33,433 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestFullRestore(234): create full backup image on multiple tables: command-line 2023-10-02T12:31:33,442 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aa9b1aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:33,448 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:33,449 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:56862, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:33,452 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:31:33,453 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:47344, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:31:33,473 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:31:33,481 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:31:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249893481 (current time:1696249893481). 2023-10-02T12:31:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:31:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:31:33,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41ffe989, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:33,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:33,498 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:56876, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:33,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:33,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:31:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=115, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:31:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 115 2023-10-02T12:31:33,505 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:31:33,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-10-02T12:31:33,506 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:31:33,513 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor129.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:31:33,516 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:31:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-10-02T12:31:33,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-10-02T12:31:33,922 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:31:33,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987}] 2023-10-02T12:31:33,929 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=116, ppid=115, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:31:34,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=116 2023-10-02T12:31:34,081 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:31:34,081 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegion(2819): Flushing 7ec576d712d0f3b0c0592584b88ce987 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-10-02T12:31:34,086 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/23e51484216f4e7a9dca73d1bb9d8f09 is 171, key is trslm:hdfs://localhost:37823/backupUT\x00ns2:test-16962497916551/meta:log-roll-map/1696249886464/Put/seqid=0 2023-10-02T12:31:34,092 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/23e51484216f4e7a9dca73d1bb9d8f09 2023-10-02T12:31:34,101 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/75d622dd3189402a9723a0642f3437f9 is 524, key is session:backup_1696249875754/session:context/1696249886513/Put/seqid=0 2023-10-02T12:31:34,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-10-02T12:31:34,510 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/75d622dd3189402a9723a0642f3437f9 2023-10-02T12:31:34,519 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/23e51484216f4e7a9dca73d1bb9d8f09 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/23e51484216f4e7a9dca73d1bb9d8f09 2023-10-02T12:31:34,528 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/23e51484216f4e7a9dca73d1bb9d8f09, entries=6, sequenceid=94, filesize=5.8 K 2023-10-02T12:31:34,530 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/75d622dd3189402a9723a0642f3437f9 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/75d622dd3189402a9723a0642f3437f9 2023-10-02T12:31:34,539 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/75d622dd3189402a9723a0642f3437f9, entries=2, sequenceid=94, filesize=5.5 K 2023-10-02T12:31:34,542 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for 7ec576d712d0f3b0c0592584b88ce987 in 461ms, sequenceid=94, compaction requested=true 2023-10-02T12:31:34,542 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegion(2520): Flush status journal for 7ec576d712d0f3b0c0592584b88ce987: 2023-10-02T12:31:34,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. for snapshot_backup_system completed. 2023-10-02T12:31:34,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:31:34,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:31:34,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/85d2568c07cc4be686b8e706b89254ce, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/9bf0c2ed3e5c43ba8bb2287a7e2731fa, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b4a8e50177b941698e5089bffe91b251, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/23e51484216f4e7a9dca73d1bb9d8f09] hfiles 2023-10-02T12:31:34,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (1/8): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98 for snapshot=snapshot_backup_system 2023-10-02T12:31:34,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (2/8): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255 for snapshot=snapshot_backup_system 2023-10-02T12:31:34,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (3/8): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8 for snapshot=snapshot_backup_system 2023-10-02T12:31:34,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (4/8): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/85d2568c07cc4be686b8e706b89254ce for snapshot=snapshot_backup_system 2023-10-02T12:31:34,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (5/8): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5 for snapshot=snapshot_backup_system 2023-10-02T12:31:34,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (6/8): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/9bf0c2ed3e5c43ba8bb2287a7e2731fa for snapshot=snapshot_backup_system 2023-10-02T12:31:34,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (7/8): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b4a8e50177b941698e5089bffe91b251 for snapshot=snapshot_backup_system 2023-10-02T12:31:34,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (8/8): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/23e51484216f4e7a9dca73d1bb9d8f09 for snapshot=snapshot_backup_system 2023-10-02T12:31:34,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/f8db732eb5cd417eaa236f25fd437245, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/fe4c59a0e2584ff5980ec23dd90e1c0b, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/2129c391bd7447f0b05fea1084f2d47d, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/120da2f382584d43adc4417dfe387fae, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/75d622dd3189402a9723a0642f3437f9] hfiles 2023-10-02T12:31:34,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (1/9): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0 for snapshot=snapshot_backup_system 2023-10-02T12:31:34,543 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (2/9): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c for snapshot=snapshot_backup_system 2023-10-02T12:31:34,544 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (3/9): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229 for snapshot=snapshot_backup_system 2023-10-02T12:31:34,544 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (4/9): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423 for snapshot=snapshot_backup_system 2023-10-02T12:31:34,544 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (5/9): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/f8db732eb5cd417eaa236f25fd437245 for snapshot=snapshot_backup_system 2023-10-02T12:31:34,544 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (6/9): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/fe4c59a0e2584ff5980ec23dd90e1c0b for snapshot=snapshot_backup_system 2023-10-02T12:31:34,544 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (7/9): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/2129c391bd7447f0b05fea1084f2d47d for snapshot=snapshot_backup_system 2023-10-02T12:31:34,544 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (8/9): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/120da2f382584d43adc4417dfe387fae for snapshot=snapshot_backup_system 2023-10-02T12:31:34,544 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (9/9): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/75d622dd3189402a9723a0642f3437f9 for snapshot=snapshot_backup_system 2023-10-02T12:31:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-10-02T12:31:34,968 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:31:34,969 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=116 2023-10-02T12:31:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=116 2023-10-02T12:31:34,969 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:31:34,971 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=116, ppid=115, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:31:34,975 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=116, resume processing ppid=115 2023-10-02T12:31:34,975 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:31:34,975 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=116, ppid=115, state=SUCCESS; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 in 1.0490 sec 2023-10-02T12:31:34,976 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:31:34,976 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:31:34,976 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:31:34,977 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:31:34,999 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:31:35,019 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:31:35,020 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:31:35,022 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:31:35,022 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 115 2023-10-02T12:31:35,024 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=115, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.5220 sec 2023-10-02T12:31:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=115 2023-10-02T12:31:35,613 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(115): Backup backup_1696249893442 started at 1696249895613. 2023-10-02T12:31:35,615 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(119): Backup session backup_1696249893442 has been started. 2023-10-02T12:31:35,621 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:31:35,622 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:31:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:31:35,622 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:31:35,622 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:31:35,624 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:31:35,624 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:31:35,625 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:31:35,626 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:35,627 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:31:35,627 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:31:35,627 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:31:35,627 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:31:35,628 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:31:35,628 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:31:35,628 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:31:35,630 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:31:35,630 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:31:35,631 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:31:35,631 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:31:35,632 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:31:35,632 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:31:35,632 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:31:35,635 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:31:35,635 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:35,635 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:31:35,635 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:31:35,636 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:31:35,636 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:31:35,636 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:31:35,636 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:35,637 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:35,637 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:31:35,637 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:31:35,638 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@16f5ec4d[Count = 0] remaining members to acquire global barrier 2023-10-02T12:31:35,638 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:31:35,638 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:31:35,639 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:31:35,639 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:31:35,639 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:31:35,639 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:31:35,639 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:31:35,639 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:35,640 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696249877989 highest: 1696249877989 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:35,640 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114:(num 1696249877989) roll requested 2023-10-02T12:31:35,655 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249895640, exclude list is [], retry=0 2023-10-02T12:31:35,658 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:31:35,665 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249877989 with entries=15, filesize=4.08 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249895640 2023-10-02T12:31:35,665 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:31:35,665 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249877989 is not closed yet, will try archiving it next time 2023-10-02T12:31:35,665 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta:.meta(num 1696249878015) roll requested 2023-10-02T12:31:35,665 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249866300 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/oldWALs/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249866300 2023-10-02T12:31:35,681 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249895665.meta, exclude list is [], retry=0 2023-10-02T12:31:35,686 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:31:35,688 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249878015.meta with entries=24, filesize=5.75 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249895665.meta 2023-10-02T12:31:35,688 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:31:35,688 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249878015.meta is not closed yet, will try archiving it next time 2023-10-02T12:31:35,691 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249878015.meta not finished, retry = 0 2023-10-02T12:31:35,700 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2023-10-02T12:31:35,700 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696249895640 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:35,708 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:31:35,708 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,38255,1696249782114' in zk 2023-10-02T12:31:35,712 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:31:35,712 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:31:35,712 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:35,712 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:31:35,712 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:31:35,713 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:31:35,713 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:31:35,714 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:31:35,714 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:35,714 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:35,715 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:31:35,715 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:35,715 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:35,716 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,38255,1696249782114': 2023-10-02T12:31:35,716 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:31:35,716 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:31:35,716 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:31:35,716 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:31:35,716 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:31:35,716 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:31:35,719 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:31:35,719 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:31:35,719 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:31:35,719 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:31:35,719 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:31:35,719 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:31:35,719 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:31:35,720 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:31:35,720 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:31:35,720 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:35,721 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:31:35,721 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:35,722 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:35,722 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:31:35,722 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:35,723 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:35,733 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:31:35,733 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:31:35,734 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:31:35,734 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:31:35,734 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:31:35,736 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:31:35,736 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:31:35,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:31:35,842 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249895841_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } 2023-10-02T12:31:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249895842 (current time:1696249895842). 2023-10-02T12:31:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:31:35,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249895841_ns2_test-16962497916551 VERSION not specified, setting to 2 2023-10-02T12:31:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a7ae7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:35,850 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:56880, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:31:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=117, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1696249895841_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } 2023-10-02T12:31:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249895841_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 }, snapshot procedure id = 117 2023-10-02T12:31:35,854 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1696249895841_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:31:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-10-02T12:31:35,855 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1696249895841_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:31:35,858 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249895841_ns2_test-16962497916551 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor129.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:31:35,862 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1696249895841_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:31:35,868 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1696249895841_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:31:35,868 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE; SnapshotRegionProcedure ce860946aa8cd08494bb77afea49e9de}] 2023-10-02T12:31:35,870 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=118, ppid=117, state=RUNNABLE; SnapshotRegionProcedure ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:31:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-10-02T12:31:36,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=118 2023-10-02T12:31:36,022 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. 2023-10-02T12:31:36,022 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.HRegion(2520): Flush status journal for ce860946aa8cd08494bb77afea49e9de: 2023-10-02T12:31:36,022 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. for snapshot_1696249895841_ns2_test-16962497916551 completed. 2023-10-02T12:31:36,022 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de.' region-info for snapshot=snapshot_1696249895841_ns2_test-16962497916551 2023-10-02T12:31:36,022 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:31:36,022 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_] hfiles 2023-10-02T12:31:36,022 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_ for snapshot=snapshot_1696249895841_ns2_test-16962497916551 2023-10-02T12:31:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-10-02T12:31:36,428 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. 2023-10-02T12:31:36,428 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=118 2023-10-02T12:31:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=118 2023-10-02T12:31:36,429 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249895841_ns2_test-16962497916551 on region ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:31:36,430 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=118, ppid=117, state=RUNNABLE; SnapshotRegionProcedure ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:31:36,432 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=118, resume processing ppid=117 2023-10-02T12:31:36,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=118, ppid=117, state=SUCCESS; SnapshotRegionProcedure ce860946aa8cd08494bb77afea49e9de in 563 msec 2023-10-02T12:31:36,433 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1696249895841_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:31:36,433 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1696249895841_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:31:36,434 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1696249895841_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:31:36,434 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249895841_ns2_test-16962497916551 2023-10-02T12:31:36,434 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249895841_ns2_test-16962497916551 2023-10-02T12:31:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-10-02T12:31:36,844 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1696249895841_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:31:36,849 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1696249895841_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:31:36,850 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249895841_ns2_test-16962497916551 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249895841_ns2_test-16962497916551 2023-10-02T12:31:36,851 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1696249895841_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:31:36,851 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249895841_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 }, snapshot procedure id = 117 2023-10-02T12:31:36,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=117, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1696249895841_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } in 999 msec 2023-10-02T12:31:36,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=117 2023-10-02T12:31:36,960 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249896959_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } 2023-10-02T12:31:36,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249896960 (current time:1696249896960). 2023-10-02T12:31:36,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:31:36,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249896959_ns3_test-16962497916552 VERSION not specified, setting to 2 2023-10-02T12:31:36,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fba0bd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:36,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:36,967 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:56892, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:36,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:36,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:31:36,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=119, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1696249896959_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } 2023-10-02T12:31:36,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249896959_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 }, snapshot procedure id = 119 2023-10-02T12:31:36,970 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1696249896959_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:31:36,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-10-02T12:31:36,971 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1696249896959_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:31:36,974 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249896959_ns3_test-16962497916552 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor129.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:31:36,976 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1696249896959_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:31:37,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-10-02T12:31:37,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-10-02T12:31:37,383 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1696249896959_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:31:37,383 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE; SnapshotRegionProcedure 9d011caaefe2252a8728679a2f592847}] 2023-10-02T12:31:37,385 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=120, ppid=119, state=RUNNABLE; SnapshotRegionProcedure 9d011caaefe2252a8728679a2f592847 2023-10-02T12:31:37,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=120 2023-10-02T12:31:37,536 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. 2023-10-02T12:31:37,536 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.HRegion(2520): Flush status journal for 9d011caaefe2252a8728679a2f592847: 2023-10-02T12:31:37,536 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. for snapshot_1696249896959_ns3_test-16962497916552 completed. 2023-10-02T12:31:37,537 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847.' region-info for snapshot=snapshot_1696249896959_ns3_test-16962497916552 2023-10-02T12:31:37,537 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:31:37,537 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:31:37,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-10-02T12:31:37,942 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. 2023-10-02T12:31:37,942 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=120 2023-10-02T12:31:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=120 2023-10-02T12:31:37,943 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249896959_ns3_test-16962497916552 on region 9d011caaefe2252a8728679a2f592847 2023-10-02T12:31:37,944 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=120, ppid=119, state=RUNNABLE; SnapshotRegionProcedure 9d011caaefe2252a8728679a2f592847 2023-10-02T12:31:37,946 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=120, resume processing ppid=119 2023-10-02T12:31:37,946 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1696249896959_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:31:37,946 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=120, ppid=119, state=SUCCESS; SnapshotRegionProcedure 9d011caaefe2252a8728679a2f592847 in 562 msec 2023-10-02T12:31:37,947 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1696249896959_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:31:37,947 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1696249896959_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:31:37,947 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249896959_ns3_test-16962497916552 2023-10-02T12:31:37,948 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249896959_ns3_test-16962497916552 2023-10-02T12:31:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-10-02T12:31:38,238 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:31:38,357 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1696249896959_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:31:38,361 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1696249896959_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:31:38,362 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249896959_ns3_test-16962497916552 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249896959_ns3_test-16962497916552 2023-10-02T12:31:38,363 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1696249896959_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:31:38,363 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249896959_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 }, snapshot procedure id = 119 2023-10-02T12:31:38,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=119, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1696249896959_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } in 1.3950 sec 2023-10-02T12:31:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=119 2023-10-02T12:31:39,077 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696249893442 2023-10-02T12:31:39,077 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:31:39,077 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-10-02T12:31:39,077 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249893442_ns2:test-16962497916551 2023-10-02T12:31:39,077 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249895841_ns2_test-16962497916551 to hdfs://localhost:37823/backupUT/backup_1696249893442/ns2/test-16962497916551/ 2023-10-02T12:31:39,077 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:31:39,113 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:31:39,113 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,40619,1696249778965_1141564007_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37823/backupUT/backup_1696249893442/ns2/test-16962497916551, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37823/backupUT/backup_1696249893442/ns2/test-16962497916551/.hbase-snapshot/.tmp/snapshot_1696249895841_ns2_test-16962497916551 2023-10-02T12:31:39,113 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07, snapshotDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249895841_ns2_test-16962497916551. 2023-10-02T12:31:39,121 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249895841_ns2_test-16962497916551 to hdfs://localhost:37823/backupUT/backup_1696249893442/ns2/test-16962497916551/.hbase-snapshot/.tmp/snapshot_1696249895841_ns2_test-16962497916551 2023-10-02T12:31:39,541 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:39,541 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:39,542 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:39,543 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:40,134 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2023-10-02T12:31:40,696 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-8550477571596864994.jar 2023-10-02T12:31:40,697 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:40,697 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:40,697 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:40,698 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:40,698 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:40,699 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:40,699 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:40,700 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:40,700 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:40,701 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:31:40,702 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:31:40,703 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:31:40,704 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:31:40,704 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:31:40,705 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:31:40,705 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:31:40,706 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:31:40,707 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:31:40,707 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:31:40,708 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:31:40,709 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:31:40,709 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:40,710 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:40,711 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:40,712 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:40,712 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:40,713 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:40,714 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:40,781 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249895841_ns2_test-16962497916551' hfile list 2023-10-02T12:31:40,784 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-10-02T12:31:40,896 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249900844/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:31:40,896 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249900844/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:31:40,981 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:31:41,015 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_ output=hdfs://localhost:37823/backupUT/backup_1696249893442/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_ 2023-10-02T12:31:41,015 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 8.321M/sec 2023-10-02T12:31:41,948 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:31:41,949 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:31:41,955 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249895841_ns2_test-16962497916551 2023-10-02T12:31:41,955 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249895841_ns2_test-16962497916551 finished. 2023-10-02T12:31:41,955 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249893442_ns3:test-16962497916552 2023-10-02T12:31:41,955 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249896959_ns3_test-16962497916552 to hdfs://localhost:37823/backupUT/backup_1696249893442/ns3/test-16962497916552/ 2023-10-02T12:31:41,955 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:31:41,981 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:31:41,981 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,40619,1696249778965_-1316311686_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37823/backupUT/backup_1696249893442/ns3/test-16962497916552, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37823/backupUT/backup_1696249893442/ns3/test-16962497916552/.hbase-snapshot/.tmp/snapshot_1696249896959_ns3_test-16962497916552 2023-10-02T12:31:41,981 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07, snapshotDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249896959_ns3_test-16962497916552. 2023-10-02T12:31:41,987 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249896959_ns3_test-16962497916552 to hdfs://localhost:37823/backupUT/backup_1696249893442/ns3/test-16962497916552/.hbase-snapshot/.tmp/snapshot_1696249896959_ns3_test-16962497916552 2023-10-02T12:31:42,006 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:42,006 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:42,007 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:42,008 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:43,167 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-1627753361833385768.jar 2023-10-02T12:31:43,168 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:43,168 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:43,168 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:43,169 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:43,169 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:43,169 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:43,170 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:43,170 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:43,170 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:43,171 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:31:43,172 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:31:43,172 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:31:43,172 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:31:43,173 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:31:43,173 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:31:43,173 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:31:43,174 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:31:43,174 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:31:43,174 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:31:43,175 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:31:43,175 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:31:43,176 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:43,176 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:43,177 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:43,177 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:43,177 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:43,178 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:43,178 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:43,223 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249896959_ns3_test-16962497916552' hfile list 2023-10-02T12:31:43,300 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249903262/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:31:43,300 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249903262/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:31:44,219 DEBUG [regionserver/jenkins-hbase3:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.46153846153846156, tune throughput to 73.08 MB/second 2023-10-02T12:31:44,329 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:31:44,331 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:31:44,337 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249896959_ns3_test-16962497916552 2023-10-02T12:31:44,337 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249896959_ns3_test-16962497916552 finished. 2023-10-02T12:31:44,337 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1034): ns2:test-16962497916551 2023-10-02T12:31:44,338 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1034): ns3:test-16962497916552 2023-10-02T12:31:44,348 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249893442 2023-10-02T12:31:44,348 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:31:44,755 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37823/backupUT/backup_1696249893442/.backup.manifest 2023-10-02T12:31:44,755 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249893442 2023-10-02T12:31:44,755 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:31:45,160 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37823/backupUT/backup_1696249893442/.backup.manifest 2023-10-02T12:31:45,161 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(358): Backup backup_1696249893442 finished: type=FULL,tablelist=ns2:test-16962497916551;ns3:test-16962497916552,targetRootDir=hdfs://localhost:37823/backupUT,startts=1696249895613,completets=1696249904348,bytescopied=0 2023-10-02T12:31:45,161 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:31:45,161 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249895841_ns2_test-16962497916551 2023-10-02T12:31:45,162 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249895841_ns2_test-16962497916551" 2023-10-02T12:31:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249895841_ns2_test-16962497916551 2023-10-02T12:31:45,165 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249895841_ns2_test-16962497916551 for backup backup_1696249893442 succeeded. 2023-10-02T12:31:45,165 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249896959_ns3_test-16962497916552 2023-10-02T12:31:45,166 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249896959_ns3_test-16962497916552" 2023-10-02T12:31:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249896959_ns3_test-16962497916552 2023-10-02T12:31:45,169 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249896959_ns3_test-16962497916552 for backup backup_1696249893442 succeeded. 2023-10-02T12:31:45,169 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:31:45,173 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:31:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:31:45,176 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:31:45,177 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:31:45,182 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(376): Backup backup_1696249893442 completed. 2023-10-02T12:31:45,183 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:45,192 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] backup.RestoreDriver(101): Found -overwrite option in restore command, will overwrite to existing table if any in the restore target 2023-10-02T12:31:45,198 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76fb0f7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:45,200 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:45,201 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37104, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:45,204 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249893442/.backup.manifest 2023-10-02T12:31:45,207 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249893442/.backup.manifest 2023-10-02T12:31:45,207 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249893442/.backup.manifest 2023-10-02T12:31:45,209 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249893442/.backup.manifest 2023-10-02T12:31:45,210 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249893442/.backup.manifest 2023-10-02T12:31:45,212 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249893442/.backup.manifest 2023-10-02T12:31:45,212 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-16962497916551' to 'ns2:test-16962497916551' from full backup image hdfs://localhost:37823/backupUT/backup_1696249893442/ns2/test-16962497916551 2023-10-02T12:31:45,220 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(476): Truncating exising target table 'ns2:test-16962497916551', preserving region splits 2023-10-02T12:31:45,221 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:31:45,222 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59720, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:31:45,223 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable ns2:test-16962497916551 2023-10-02T12:31:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=121, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:test-16962497916551 2023-10-02T12:31:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=121 2023-10-02T12:31:45,228 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962497916551","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249905227"}]},"ts":"1696249905227"} 2023-10-02T12:31:45,229 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962497916551, state=DISABLING in hbase:meta 2023-10-02T12:31:45,230 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set ns2:test-16962497916551 to state=DISABLING 2023-10-02T12:31:45,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-16962497916551, region=ce860946aa8cd08494bb77afea49e9de, UNASSIGN}] 2023-10-02T12:31:45,233 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=122, ppid=121, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-16962497916551, region=ce860946aa8cd08494bb77afea49e9de, UNASSIGN 2023-10-02T12:31:45,233 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=122 updating hbase:meta row=ce860946aa8cd08494bb77afea49e9de, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:45,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE; CloseRegionProcedure ce860946aa8cd08494bb77afea49e9de, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:31:45,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=121 2023-10-02T12:31:45,385 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] handler.UnassignRegionHandler(119): Close ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:31:45,385 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1668): Closing ce860946aa8cd08494bb77afea49e9de, disabling compactions & flushes 2023-10-02T12:31:45,385 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1690): Closing region ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. 2023-10-02T12:31:45,385 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. 2023-10-02T12:31:45,386 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. after waiting 0 ms 2023-10-02T12:31:45,386 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. 2023-10-02T12:31:45,392 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2023-10-02T12:31:45,392 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:31:45,393 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1909): Closed ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de. 2023-10-02T12:31:45,393 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1622): Region close journal for ce860946aa8cd08494bb77afea49e9de: 2023-10-02T12:31:45,394 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] handler.UnassignRegionHandler(163): Closed ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:31:45,395 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=122 updating hbase:meta row=ce860946aa8cd08494bb77afea49e9de, regionState=CLOSED 2023-10-02T12:31:45,397 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=123, resume processing ppid=122 2023-10-02T12:31:45,397 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=123, ppid=122, state=SUCCESS; CloseRegionProcedure ce860946aa8cd08494bb77afea49e9de, server=jenkins-hbase3.apache.org,38255,1696249782114 in 162 msec 2023-10-02T12:31:45,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=122, resume processing ppid=121 2023-10-02T12:31:45,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=122, ppid=121, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16962497916551, region=ce860946aa8cd08494bb77afea49e9de, UNASSIGN in 166 msec 2023-10-02T12:31:45,399 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962497916551","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249905399"}]},"ts":"1696249905399"} 2023-10-02T12:31:45,400 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962497916551, state=DISABLED in hbase:meta 2023-10-02T12:31:45,402 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set ns2:test-16962497916551 to state=DISABLED 2023-10-02T12:31:45,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=121, state=SUCCESS; DisableTableProcedure table=ns2:test-16962497916551 in 179 msec 2023-10-02T12:31:45,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=121 2023-10-02T12:31:45,531 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$6(2480): Client=jenkins//172.31.12.81 truncate ns2:test-16962497916551 2023-10-02T12:31:45,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=124, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns2:test-16962497916551 preserveSplits=true) 2023-10-02T12:31:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=124 2023-10-02T12:31:45,534 DEBUG [PEWorker-5 {}] procedure.TruncateTableProcedure(87): waiting for 'ns2:test-16962497916551' regions in transition 2023-10-02T12:31:45,536 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:31:45,538 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f, FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/recovered.edits] 2023-10-02T12:31:45,543 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_ to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/archive/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_ 2023-10-02T12:31:45,549 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/recovered.edits/9.seqid to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/archive/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/recovered.edits/9.seqid 2023-10-02T12:31:45,550 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:31:45,550 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived ns2:test-16962497916551 regions 2023-10-02T12:31:45,555 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:test-16962497916551 from hbase:meta 2023-10-02T12:31:45,557 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:test-16962497916551' descriptor. 2023-10-02T12:31:45,558 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:test-16962497916551' from region states. 2023-10-02T12:31:45,558 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696249905558"}]},"ts":"9223372036854775807"} 2023-10-02T12:31:45,560 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:31:45,560 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => ce860946aa8cd08494bb77afea49e9de, NAME => 'ns2:test-16962497916551,,1696249813426.ce860946aa8cd08494bb77afea49e9de.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:31:45,560 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:test-16962497916551' as deleted. 2023-10-02T12:31:45,560 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-16962497916551","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1696249905560"}]},"ts":"9223372036854775807"} 2023-10-02T12:31:45,562 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:test-16962497916551 state from META 2023-10-02T12:31:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=124 2023-10-02T12:31:45,639 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:31:45,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=124 2023-10-02T12:31:45,972 INFO [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 1ff8cee0e4d9c23004d17b9d828c2c96, NAME => 'ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-16962497916551', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:31:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=124 2023-10-02T12:31:46,378 DEBUG [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:31:46,378 DEBUG [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1668): Closing 1ff8cee0e4d9c23004d17b9d828c2c96, disabling compactions & flushes 2023-10-02T12:31:46,378 INFO [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96. 2023-10-02T12:31:46,378 DEBUG [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96. 2023-10-02T12:31:46,378 DEBUG [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96. after waiting 0 ms 2023-10-02T12:31:46,378 DEBUG [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96. 2023-10-02T12:31:46,378 INFO [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96. 2023-10-02T12:31:46,378 DEBUG [RegionOpenAndInit-ns2:test-16962497916551-pool-0 {}] regionserver.HRegion(1622): Region close journal for 1ff8cee0e4d9c23004d17b9d828c2c96: 2023-10-02T12:31:46,380 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696249906379"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249906379"}]},"ts":"1696249906379"} 2023-10-02T12:31:46,381 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:31:46,381 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962497916551","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249906381"}]},"ts":"1696249906381"} 2023-10-02T12:31:46,382 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962497916551, state=ENABLING in hbase:meta 2023-10-02T12:31:46,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16962497916551, region=1ff8cee0e4d9c23004d17b9d828c2c96, ASSIGN}] 2023-10-02T12:31:46,389 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-16962497916551, region=1ff8cee0e4d9c23004d17b9d828c2c96, ASSIGN 2023-10-02T12:31:46,390 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-16962497916551, region=1ff8cee0e4d9c23004d17b9d828c2c96, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38255,1696249782114; forceNewPlan=false, retain=false 2023-10-02T12:31:46,541 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=125 updating hbase:meta row=1ff8cee0e4d9c23004d17b9d828c2c96, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:46,543 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE; OpenRegionProcedure 1ff8cee0e4d9c23004d17b9d828c2c96, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:31:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=124 2023-10-02T12:31:46,698 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(130): Open ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96. 2023-10-02T12:31:46,698 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7254): Opening region: {ENCODED => 1ff8cee0e4d9c23004d17b9d828c2c96, NAME => 'ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:31:46,699 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:31:46,699 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962497916551 1ff8cee0e4d9c23004d17b9d828c2c96 2023-10-02T12:31:46,699 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(888): Instantiated ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:31:46,699 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7296): checking encryption for 1ff8cee0e4d9c23004d17b9d828c2c96 2023-10-02T12:31:46,699 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7299): checking classloading for 1ff8cee0e4d9c23004d17b9d828c2c96 2023-10-02T12:31:46,700 INFO [StoreOpener-1ff8cee0e4d9c23004d17b9d828c2c96-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 1ff8cee0e4d9c23004d17b9d828c2c96 2023-10-02T12:31:46,701 INFO [StoreOpener-1ff8cee0e4d9c23004d17b9d828c2c96-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ff8cee0e4d9c23004d17b9d828c2c96 columnFamilyName f 2023-10-02T12:31:46,702 DEBUG [StoreOpener-1ff8cee0e4d9c23004d17b9d828c2c96-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:31:46,702 INFO [StoreOpener-1ff8cee0e4d9c23004d17b9d828c2c96-1 {}] regionserver.HStore(324): Store=1ff8cee0e4d9c23004d17b9d828c2c96/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:31:46,703 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96 2023-10-02T12:31:46,703 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96 2023-10-02T12:31:46,705 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1076): writing seq id for 1ff8cee0e4d9c23004d17b9d828c2c96 2023-10-02T12:31:46,706 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:31:46,707 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): Opened 1ff8cee0e4d9c23004d17b9d828c2c96; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11895834560, jitterRate=0.1078859269618988}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:31:46,707 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(992): Region open journal for 1ff8cee0e4d9c23004d17b9d828c2c96: 2023-10-02T12:31:46,708 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96., pid=126, masterSystemTime=1696249906694 2023-10-02T12:31:46,709 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96. 2023-10-02T12:31:46,710 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(158): Opened ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96. 2023-10-02T12:31:46,710 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=125 updating hbase:meta row=1ff8cee0e4d9c23004d17b9d828c2c96, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:46,713 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=126, resume processing ppid=125 2023-10-02T12:31:46,713 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=126, ppid=125, state=SUCCESS; OpenRegionProcedure 1ff8cee0e4d9c23004d17b9d828c2c96, server=jenkins-hbase3.apache.org,38255,1696249782114 in 168 msec 2023-10-02T12:31:46,714 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=125, resume processing ppid=124 2023-10-02T12:31:46,714 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=125, ppid=124, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-16962497916551, region=1ff8cee0e4d9c23004d17b9d828c2c96, ASSIGN in 325 msec 2023-10-02T12:31:46,714 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-16962497916551","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249906714"}]},"ts":"1696249906714"} 2023-10-02T12:31:46,715 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-16962497916551, state=ENABLED in hbase:meta 2023-10-02T12:31:46,717 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(145): truncate 'ns2:test-16962497916551' completed 2023-10-02T12:31:46,718 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=124, state=SUCCESS; TruncateTableProcedure (table=ns2:test-16962497916551 preserveSplits=true) in 1.1860 sec 2023-10-02T12:31:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=124 2023-10-02T12:31:47,645 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:37823/backupUT/backup_1696249893442/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de from hbase tables ns2:test-16962497916551 to tables ns2:test-16962497916551 2023-10-02T12:31:47,645 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-16962497916551 into ns2:test-16962497916551 2023-10-02T12:31:47,647 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-test-16962497916551-1696249907645 from hdfs://localhost:37823/backupUT/backup_1696249893442/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de 2023-10-02T12:31:47,652 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e4c0df4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:47,652 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:31:47,654 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59728, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:31:47,655 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-10-02T12:31:47,655 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-16962497916551 2023-10-02T12:31:47,658 DEBUG [hconnection-0x3b815f87-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:47,659 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37106, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:47,660 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-16962497916551: 2023-10-02T12:31:47,660 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-10-02T12:31:47,660 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/partitions_e3f0d760-8d34-4569-bc79-ea5e303678f3 2023-10-02T12:31:47,666 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:47,667 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:47,668 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:47,668 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:48,671 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-6056233770519355445.jar 2023-10-02T12:31:48,671 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:48,672 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:48,672 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:48,672 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:48,673 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:48,673 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:48,673 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:48,674 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:48,674 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:48,675 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:31:48,675 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:31:48,676 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:31:48,677 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:31:48,677 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:31:48,678 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:31:48,678 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:31:48,679 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:31:48,680 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:31:48,680 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:31:48,681 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:31:48,681 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:31:48,682 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:48,682 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:48,683 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:48,683 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:48,683 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:48,683 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:48,684 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:48,684 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-16962497916551 2023-10-02T12:31:48,685 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:48,685 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-10-02T12:31:48,685 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-10-02T12:31:48,686 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:31:48,697 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-10-02T12:31:48,801 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249908753/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:31:48,801 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249908753/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:31:48,851 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:37823/backupUT/backup_1696249893442/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/ce860946aa8cd08494bb77afea49e9de/f/b73dd01443e64117bbdffc3ea354d298_SeqId_4_ 2023-10-02T12:31:48,856 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-10-02T12:31:48,884 DEBUG [pool-440-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd6ef70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:48,885 DEBUG [hconnection-0xa381acb-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:48,887 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37110, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:48,891 DEBUG [pool-440-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:48,891 DEBUG [pool-440-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-10-02T12:31:48,892 DEBUG [pool-440-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-10-02T12:31:48,899 DEBUG [pool-440-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-test-16962497916551-1696249907645/_temporary/0/_temporary/attempt_local1541208311_0019_r_000000_0/f/4027449e08a2487ba503f7928e7e8079 is 37, key is row10/f:q1/1696249795545/Put/seqid=0 2023-10-02T12:31:48,903 WARN [Thread-1913 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:38255] Specified favored nodes: [jenkins-hbase3.apache.org:38255] 2023-10-02T12:31:49,853 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-test-16962497916551-1696249907645 2023-10-02T12:31:49,859 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15da0fdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:49,861 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:49,862 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37126, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:49,870 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-test-16962497916551-1696249907645/_SUCCESS 2023-10-02T12:31:49,875 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:31:49,877 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59736, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:31:49,888 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-test-16962497916551-1696249907645/f/4027449e08a2487ba503f7928e7e8079 first=Optional[row0] last=Optional[row98] 2023-10-02T12:31:49,890 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96., hostname=jenkins-hbase3.apache.org,38255,1696249782114, seqNum=2 for row with hfile group [{f,hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-test-16962497916551-1696249907645/f/4027449e08a2487ba503f7928e7e8079}] 2023-10-02T12:31:49,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-test-16962497916551-1696249907645/f/4027449e08a2487ba503f7928e7e8079 for inclusion in 1ff8cee0e4d9c23004d17b9d828c2c96/f 2023-10-02T12:31:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-10-02T12:31:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(625): Region bounds: first= last= 2023-10-02T12:31:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HRegion(2520): Flush status journal for 1ff8cee0e4d9c23004d17b9d828c2c96: 2023-10-02T12:31:49,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-test-16962497916551-1696249907645/f/4027449e08a2487ba503f7928e7e8079 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__ns2_test-16962497916551__9a8bge0am955rsvk03k4il214ctvg3lhequtivm3tc7amt3geh4guoelpei1dvh3/f/4027449e08a2487ba503f7928e7e8079 2023-10-02T12:31:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__ns2_test-16962497916551__9a8bge0am955rsvk03k4il214ctvg3lhequtivm3tc7amt3geh4guoelpei1dvh3/f/4027449e08a2487ba503f7928e7e8079 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96/f/eec10f547780452b96a08ad7a8ce56ca_SeqId_4_ 2023-10-02T12:31:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c0b2afd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:31:49,922 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59740, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:31:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:49,928 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37142, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(428): write bulk load descriptor to backup ns2:test-16962497916551 with 1 entries 2023-10-02T12:31:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(1602): writing raw bulk path hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96/f/eec10f547780452b96a08ad7a8ce56ca_SeqId_4_ for ns2:test-16962497916551 1ff8cee0e4d9c23004d17b9d828c2c96 2023-10-02T12:31:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(435): written 1 rows for bulk load of ns2:test-16962497916551 2023-10-02T12:31:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:49,964 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__ns2_test-16962497916551__9a8bge0am955rsvk03k4il214ctvg3lhequtivm3tc7amt3geh4guoelpei1dvh3/f/4027449e08a2487ba503f7928e7e8079 into 1ff8cee0e4d9c23004d17b9d828c2c96/f as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96/f/eec10f547780452b96a08ad7a8ce56ca_SeqId_4_ - updating store file list. 2023-10-02T12:31:49,971 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96/f/eec10f547780452b96a08ad7a8ce56ca_SeqId_4_ into 1ff8cee0e4d9c23004d17b9d828c2c96/f 2023-10-02T12:31:49,971 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__ns2_test-16962497916551__9a8bge0am955rsvk03k4il214ctvg3lhequtivm3tc7amt3geh4guoelpei1dvh3/f/4027449e08a2487ba503f7928e7e8079 into 1ff8cee0e4d9c23004d17b9d828c2c96/f (new location: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96/f/eec10f547780452b96a08ad7a8ce56ca_SeqId_4_) 2023-10-02T12:31:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__ns2_test-16962497916551__9a8bge0am955rsvk03k4il214ctvg3lhequtivm3tc7amt3geh4guoelpei1dvh3/f/4027449e08a2487ba503f7928e7e8079 2023-10-02T12:31:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@516dbd52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:50,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:31:50,070 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:59742, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-10-02T12:31:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:50,075 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:37144, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-10-02T12:31:50,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup ns2:test-16962497916551 with 1 entries 2023-10-02T12:31:50,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(1523): writing done bulk path hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96/f/eec10f547780452b96a08ad7a8ce56ca_SeqId_4_ for ns2:test-16962497916551 1ff8cee0e4d9c23004d17b9d828c2c96 2023-10-02T12:31:50,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] impl.BackupSystemTable(414): written 1 rows for bulk load of ns2:test-16962497916551 2023-10-02T12:31:50,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:50,095 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:50,095 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-10-02T12:31:50,096 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-10-02T12:31:50,096 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(233): Backup: backup_1696249893442 hdfs://localhost:37823/backupUT/backup_1696249893442/ns2/test-16962497916551/ 2023-10-02T12:31:50,096 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249893442/.backup.manifest 2023-10-02T12:31:50,099 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249893442/.backup.manifest 2023-10-02T12:31:50,099 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-16962497916552' to 'ns3:test-16962497916552' from full backup image hdfs://localhost:37823/backupUT/backup_1696249893442/ns3/test-16962497916552 2023-10-02T12:31:50,106 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:37823/backupUT/backup_1696249893442/ns3/test-16962497916552/archive/data/ns3/test-16962497916552 does not exists 2023-10-02T12:31:50,106 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-16962497916552, will only create table 2023-10-02T12:31:50,107 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(476): Truncating exising target table 'ns3:test-16962497916552', preserving region splits 2023-10-02T12:31:50,108 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable ns3:test-16962497916552 2023-10-02T12:31:50,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=127, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:test-16962497916552 2023-10-02T12:31:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=127 2023-10-02T12:31:50,113 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962497916552","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249910112"}]},"ts":"1696249910112"} 2023-10-02T12:31:50,114 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962497916552, state=DISABLING in hbase:meta 2023-10-02T12:31:50,199 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set ns3:test-16962497916552 to state=DISABLING 2023-10-02T12:31:50,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-16962497916552, region=9d011caaefe2252a8728679a2f592847, UNASSIGN}] 2023-10-02T12:31:50,202 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=128, ppid=127, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-16962497916552, region=9d011caaefe2252a8728679a2f592847, UNASSIGN 2023-10-02T12:31:50,202 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=128 updating hbase:meta row=9d011caaefe2252a8728679a2f592847, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:50,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE; CloseRegionProcedure 9d011caaefe2252a8728679a2f592847, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:31:50,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=127 2023-10-02T12:31:50,356 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] handler.UnassignRegionHandler(119): Close 9d011caaefe2252a8728679a2f592847 2023-10-02T12:31:50,356 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1668): Closing 9d011caaefe2252a8728679a2f592847, disabling compactions & flushes 2023-10-02T12:31:50,356 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1690): Closing region ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. 2023-10-02T12:31:50,356 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. 2023-10-02T12:31:50,357 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. after waiting 0 ms 2023-10-02T12:31:50,357 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. 2023-10-02T12:31:50,361 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/9d011caaefe2252a8728679a2f592847/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=1 2023-10-02T12:31:50,362 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:31:50,363 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1909): Closed ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847. 2023-10-02T12:31:50,363 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1622): Region close journal for 9d011caaefe2252a8728679a2f592847: 2023-10-02T12:31:50,366 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] handler.UnassignRegionHandler(163): Closed 9d011caaefe2252a8728679a2f592847 2023-10-02T12:31:50,367 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=128 updating hbase:meta row=9d011caaefe2252a8728679a2f592847, regionState=CLOSED 2023-10-02T12:31:50,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=129, resume processing ppid=128 2023-10-02T12:31:50,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=129, ppid=128, state=SUCCESS; CloseRegionProcedure 9d011caaefe2252a8728679a2f592847, server=jenkins-hbase3.apache.org,38255,1696249782114 in 164 msec 2023-10-02T12:31:50,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=128, resume processing ppid=127 2023-10-02T12:31:50,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=128, ppid=127, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16962497916552, region=9d011caaefe2252a8728679a2f592847, UNASSIGN in 170 msec 2023-10-02T12:31:50,372 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962497916552","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249910372"}]},"ts":"1696249910372"} 2023-10-02T12:31:50,373 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962497916552, state=DISABLED in hbase:meta 2023-10-02T12:31:50,380 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set ns3:test-16962497916552 to state=DISABLED 2023-10-02T12:31:50,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=127, state=SUCCESS; DisableTableProcedure table=ns3:test-16962497916552 in 273 msec 2023-10-02T12:31:50,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=127 2023-10-02T12:31:50,419 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$6(2480): Client=jenkins//172.31.12.81 truncate ns3:test-16962497916552 2023-10-02T12:31:50,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=130, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns3:test-16962497916552 preserveSplits=true) 2023-10-02T12:31:50,422 DEBUG [PEWorker-3 {}] procedure.TruncateTableProcedure(87): waiting for 'ns3:test-16962497916552' regions in transition 2023-10-02T12:31:50,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=130 2023-10-02T12:31:50,424 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/9d011caaefe2252a8728679a2f592847 2023-10-02T12:31:50,426 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/9d011caaefe2252a8728679a2f592847/f, FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/9d011caaefe2252a8728679a2f592847/recovered.edits] 2023-10-02T12:31:50,434 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/9d011caaefe2252a8728679a2f592847/recovered.edits/7.seqid to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/archive/data/ns3/test-16962497916552/9d011caaefe2252a8728679a2f592847/recovered.edits/7.seqid 2023-10-02T12:31:50,434 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/9d011caaefe2252a8728679a2f592847 2023-10-02T12:31:50,435 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(293): Archived ns3:test-16962497916552 regions 2023-10-02T12:31:50,440 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:test-16962497916552 from hbase:meta 2023-10-02T12:31:50,443 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:test-16962497916552' descriptor. 2023-10-02T12:31:50,443 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:test-16962497916552' from region states. 2023-10-02T12:31:50,444 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696249910443"}]},"ts":"9223372036854775807"} 2023-10-02T12:31:50,446 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:31:50,446 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 9d011caaefe2252a8728679a2f592847, NAME => 'ns3:test-16962497916552,,1696249817477.9d011caaefe2252a8728679a2f592847.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:31:50,446 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:test-16962497916552' as deleted. 2023-10-02T12:31:50,446 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-16962497916552","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1696249910446"}]},"ts":"9223372036854775807"} 2023-10-02T12:31:50,449 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:test-16962497916552 state from META 2023-10-02T12:31:50,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=130 2023-10-02T12:31:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=130 2023-10-02T12:31:50,874 INFO [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 8f9314653c5767b76ceff134e893c29a, NAME => 'ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-16962497916552', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:31:50,896 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:31:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=130 2023-10-02T12:31:51,361 DEBUG [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:31:51,361 DEBUG [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1668): Closing 8f9314653c5767b76ceff134e893c29a, disabling compactions & flushes 2023-10-02T12:31:51,361 INFO [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a. 2023-10-02T12:31:51,361 DEBUG [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a. 2023-10-02T12:31:51,361 DEBUG [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a. after waiting 0 ms 2023-10-02T12:31:51,361 DEBUG [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a. 2023-10-02T12:31:51,361 INFO [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a. 2023-10-02T12:31:51,361 DEBUG [RegionOpenAndInit-ns3:test-16962497916552-pool-0 {}] regionserver.HRegion(1622): Region close journal for 8f9314653c5767b76ceff134e893c29a: 2023-10-02T12:31:51,363 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1696249911363"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249911363"}]},"ts":"1696249911363"} 2023-10-02T12:31:51,365 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:31:51,366 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962497916552","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249911365"}]},"ts":"1696249911365"} 2023-10-02T12:31:51,369 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962497916552, state=ENABLING in hbase:meta 2023-10-02T12:31:51,373 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16962497916552, region=8f9314653c5767b76ceff134e893c29a, ASSIGN}] 2023-10-02T12:31:51,377 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-16962497916552, region=8f9314653c5767b76ceff134e893c29a, ASSIGN 2023-10-02T12:31:51,378 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-16962497916552, region=8f9314653c5767b76ceff134e893c29a, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38255,1696249782114; forceNewPlan=false, retain=false 2023-10-02T12:31:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=130 2023-10-02T12:31:51,530 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=131 updating hbase:meta row=8f9314653c5767b76ceff134e893c29a, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:51,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE; OpenRegionProcedure 8f9314653c5767b76ceff134e893c29a, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:31:51,688 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(130): Open ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a. 2023-10-02T12:31:51,688 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7254): Opening region: {ENCODED => 8f9314653c5767b76ceff134e893c29a, NAME => 'ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:31:51,688 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:31:51,688 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-16962497916552 8f9314653c5767b76ceff134e893c29a 2023-10-02T12:31:51,688 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(888): Instantiated ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:31:51,689 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7296): checking encryption for 8f9314653c5767b76ceff134e893c29a 2023-10-02T12:31:51,689 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7299): checking classloading for 8f9314653c5767b76ceff134e893c29a 2023-10-02T12:31:51,691 INFO [StoreOpener-8f9314653c5767b76ceff134e893c29a-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 8f9314653c5767b76ceff134e893c29a 2023-10-02T12:31:51,693 INFO [StoreOpener-8f9314653c5767b76ceff134e893c29a-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f9314653c5767b76ceff134e893c29a columnFamilyName f 2023-10-02T12:31:51,693 DEBUG [StoreOpener-8f9314653c5767b76ceff134e893c29a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:31:51,694 INFO [StoreOpener-8f9314653c5767b76ceff134e893c29a-1 {}] regionserver.HStore(324): Store=8f9314653c5767b76ceff134e893c29a/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:31:51,695 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/8f9314653c5767b76ceff134e893c29a 2023-10-02T12:31:51,695 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/8f9314653c5767b76ceff134e893c29a 2023-10-02T12:31:51,697 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1076): writing seq id for 8f9314653c5767b76ceff134e893c29a 2023-10-02T12:31:51,700 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/8f9314653c5767b76ceff134e893c29a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:31:51,701 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1093): Opened 8f9314653c5767b76ceff134e893c29a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9685284320, jitterRate=-0.09798760712146759}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:31:51,702 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(992): Region open journal for 8f9314653c5767b76ceff134e893c29a: 2023-10-02T12:31:51,703 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a., pid=132, masterSystemTime=1696249911684 2023-10-02T12:31:51,704 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a. 2023-10-02T12:31:51,704 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(158): Opened ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a. 2023-10-02T12:31:51,705 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=131 updating hbase:meta row=8f9314653c5767b76ceff134e893c29a, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:51,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=132, resume processing ppid=131 2023-10-02T12:31:51,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=132, ppid=131, state=SUCCESS; OpenRegionProcedure 8f9314653c5767b76ceff134e893c29a, server=jenkins-hbase3.apache.org,38255,1696249782114 in 174 msec 2023-10-02T12:31:51,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=131, resume processing ppid=130 2023-10-02T12:31:51,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=131, ppid=130, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-16962497916552, region=8f9314653c5767b76ceff134e893c29a, ASSIGN in 335 msec 2023-10-02T12:31:51,710 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-16962497916552","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249911710"}]},"ts":"1696249911710"} 2023-10-02T12:31:51,718 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-16962497916552, state=ENABLED in hbase:meta 2023-10-02T12:31:51,722 DEBUG [PEWorker-3 {}] procedure.TruncateTableProcedure(145): truncate 'ns3:test-16962497916552' completed 2023-10-02T12:31:51,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=130, state=SUCCESS; TruncateTableProcedure (table=ns3:test-16962497916552 preserveSplits=true) in 1.3030 sec 2023-10-02T12:31:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=130 2023-10-02T12:31:52,536 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-10-02T12:31:52,536 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(233): Backup: backup_1696249893442 hdfs://localhost:37823/backupUT/backup_1696249893442/ns3/test-16962497916552/ 2023-10-02T12:31:52,536 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-10-02T12:31:52,537 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:52,564 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleOverwriteCommand Thread=606 (was 600) - Thread LEAK? -, OpenFileDescriptor=683 (was 680) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=539 (was 550), ProcessCount=181 (was 181), AvailableMemoryMB=2898 (was 1855) - AvailableMemoryMB LEAK? - 2023-10-02T12:31:52,564 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=606 is superior to 500 2023-10-02T12:31:52,584 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleCommand Thread=606, OpenFileDescriptor=683, MaxFileDescriptor=60000, SystemLoadAverage=539, ProcessCount=181, AvailableMemoryMB=2896 2023-10-02T12:31:52,584 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=606 is superior to 500 2023-10-02T12:31:52,584 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestFullRestore(144): create full backup image on multiple tables: command-line 2023-10-02T12:31:52,589 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37680f97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:52,593 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:52,594 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60994, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:52,612 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:31:52,613 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:60708, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:31:52,652 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:31:52,665 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:31:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249912665 (current time:1696249912665). 2023-10-02T12:31:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:31:52,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:31:52,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c46e0a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:52,695 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:32778, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:31:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=133, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:31:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 133 2023-10-02T12:31:52,700 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:31:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=133 2023-10-02T12:31:52,700 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:31:52,704 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor129.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:31:52,708 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:31:52,719 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:31:52,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987}] 2023-10-02T12:31:52,722 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=134, ppid=133, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:31:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=133 2023-10-02T12:31:52,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=134 2023-10-02T12:31:52,873 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:31:52,874 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegion(2819): Flushing 7ec576d712d0f3b0c0592584b88ce987 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-10-02T12:31:52,878 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/f1ceb39b31074c03be29750a860f2cc8 is 171, key is trslm:hdfs://localhost:37823/backupUT\x00ns2:test-16962497916551/meta:log-roll-map/1696249904344/Put/seqid=0 2023-10-02T12:31:53,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=133 2023-10-02T12:31:53,296 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/f1ceb39b31074c03be29750a860f2cc8 2023-10-02T12:31:53,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=133 2023-10-02T12:31:53,308 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/56ef49f7bed44dd1b83792cad64956f0 is 524, key is session:backup_1696249893442/session:context/1696249905176/Put/seqid=0 2023-10-02T12:31:53,713 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/56ef49f7bed44dd1b83792cad64956f0 2023-10-02T12:31:53,720 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/f1ceb39b31074c03be29750a860f2cc8 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/f1ceb39b31074c03be29750a860f2cc8 2023-10-02T12:31:53,725 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/f1ceb39b31074c03be29750a860f2cc8, entries=6, sequenceid=105, filesize=5.8 K 2023-10-02T12:31:53,726 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/56ef49f7bed44dd1b83792cad64956f0 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/56ef49f7bed44dd1b83792cad64956f0 2023-10-02T12:31:53,731 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/56ef49f7bed44dd1b83792cad64956f0, entries=2, sequenceid=105, filesize=5.5 K 2023-10-02T12:31:53,732 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for 7ec576d712d0f3b0c0592584b88ce987 in 858ms, sequenceid=105, compaction requested=true 2023-10-02T12:31:53,732 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegion(2520): Flush status journal for 7ec576d712d0f3b0c0592584b88ce987: 2023-10-02T12:31:53,732 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. for snapshot_backup_system completed. 2023-10-02T12:31:53,732 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:31:53,732 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:31:53,732 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/85d2568c07cc4be686b8e706b89254ce, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/9bf0c2ed3e5c43ba8bb2287a7e2731fa, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b4a8e50177b941698e5089bffe91b251, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/23e51484216f4e7a9dca73d1bb9d8f09, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/f1ceb39b31074c03be29750a860f2cc8] hfiles 2023-10-02T12:31:53,732 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (1/9): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98 for snapshot=snapshot_backup_system 2023-10-02T12:31:53,732 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (2/9): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255 for snapshot=snapshot_backup_system 2023-10-02T12:31:53,732 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (3/9): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8 for snapshot=snapshot_backup_system 2023-10-02T12:31:53,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (4/9): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/85d2568c07cc4be686b8e706b89254ce for snapshot=snapshot_backup_system 2023-10-02T12:31:53,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (5/9): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5 for snapshot=snapshot_backup_system 2023-10-02T12:31:53,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (6/9): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/9bf0c2ed3e5c43ba8bb2287a7e2731fa for snapshot=snapshot_backup_system 2023-10-02T12:31:53,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (7/9): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b4a8e50177b941698e5089bffe91b251 for snapshot=snapshot_backup_system 2023-10-02T12:31:53,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (8/9): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/23e51484216f4e7a9dca73d1bb9d8f09 for snapshot=snapshot_backup_system 2023-10-02T12:31:53,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (9/9): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/f1ceb39b31074c03be29750a860f2cc8 for snapshot=snapshot_backup_system 2023-10-02T12:31:53,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/f8db732eb5cd417eaa236f25fd437245, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/fe4c59a0e2584ff5980ec23dd90e1c0b, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/2129c391bd7447f0b05fea1084f2d47d, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/120da2f382584d43adc4417dfe387fae, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/75d622dd3189402a9723a0642f3437f9, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/56ef49f7bed44dd1b83792cad64956f0] hfiles 2023-10-02T12:31:53,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (1/10): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0 for snapshot=snapshot_backup_system 2023-10-02T12:31:53,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (2/10): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c for snapshot=snapshot_backup_system 2023-10-02T12:31:53,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (3/10): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229 for snapshot=snapshot_backup_system 2023-10-02T12:31:53,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (4/10): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423 for snapshot=snapshot_backup_system 2023-10-02T12:31:53,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (5/10): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/f8db732eb5cd417eaa236f25fd437245 for snapshot=snapshot_backup_system 2023-10-02T12:31:53,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (6/10): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/fe4c59a0e2584ff5980ec23dd90e1c0b for snapshot=snapshot_backup_system 2023-10-02T12:31:53,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (7/10): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/2129c391bd7447f0b05fea1084f2d47d for snapshot=snapshot_backup_system 2023-10-02T12:31:53,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (8/10): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/120da2f382584d43adc4417dfe387fae for snapshot=snapshot_backup_system 2023-10-02T12:31:53,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (9/10): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/75d622dd3189402a9723a0642f3437f9 for snapshot=snapshot_backup_system 2023-10-02T12:31:53,733 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (10/10): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/56ef49f7bed44dd1b83792cad64956f0 for snapshot=snapshot_backup_system 2023-10-02T12:31:53,755 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:31:53,755 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=134 2023-10-02T12:31:53,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=134 2023-10-02T12:31:53,756 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:31:53,757 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=134, ppid=133, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:31:53,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=134, resume processing ppid=133 2023-10-02T12:31:53,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=134, ppid=133, state=SUCCESS; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 in 1.0390 sec 2023-10-02T12:31:53,759 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:31:53,760 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:31:53,760 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:31:53,760 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:31:53,763 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:31:53,790 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:31:53,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=133 2023-10-02T12:31:53,846 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:31:53,847 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:31:53,848 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:31:53,848 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 133 2023-10-02T12:31:53,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=133, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.1510 sec 2023-10-02T12:31:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=133 2023-10-02T12:31:54,840 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(115): Backup backup_1696249912589 started at 1696249914840. 2023-10-02T12:31:54,843 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(119): Backup session backup_1696249912589 has been started. 2023-10-02T12:31:54,851 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:31:54,852 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:31:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:31:54,853 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:31:54,853 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:31:54,854 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:31:54,854 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:31:54,855 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:31:54,857 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:54,857 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:31:54,857 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:31:54,857 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:31:54,857 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:31:54,858 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:31:54,858 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:31:54,858 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:31:54,864 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:31:54,864 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:31:54,864 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:31:54,864 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:31:54,868 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:31:54,868 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:31:54,868 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:31:54,871 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:31:54,871 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:54,871 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:31:54,871 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:31:54,871 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:31:54,872 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:31:54,872 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:31:54,872 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:54,873 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:54,873 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:31:54,873 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:31:54,874 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@5707c436[Count = 0] remaining members to acquire global barrier 2023-10-02T12:31:54,874 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:31:54,874 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:31:54,877 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:31:54,877 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:31:54,877 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:31:54,878 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:31:54,878 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:31:54,878 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:54,878 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696249895640 highest: 1696249895640 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:54,878 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114:(num 1696249895640) roll requested 2023-10-02T12:31:54,895 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249914878, exclude list is [], retry=0 2023-10-02T12:31:54,902 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:31:54,906 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249895640 with entries=17, filesize=5.81 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249914878 2023-10-02T12:31:54,906 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:31:54,906 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249895640 is not closed yet, will try archiving it next time 2023-10-02T12:31:54,907 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta:.meta(num 1696249895665) roll requested 2023-10-02T12:31:54,907 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249877989 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/oldWALs/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249877989 2023-10-02T12:31:54,924 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249914907.meta, exclude list is [], retry=0 2023-10-02T12:31:54,929 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:31:54,937 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249895665.meta with entries=24, filesize=6.28 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249914907.meta 2023-10-02T12:31:54,937 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:31:54,937 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249895665.meta is not closed yet, will try archiving it next time 2023-10-02T12:31:54,941 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 63 2023-10-02T12:31:54,941 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696249914878 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:54,956 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:31:54,956 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,38255,1696249782114' in zk 2023-10-02T12:31:54,960 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:31:54,960 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:31:54,960 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:31:54,960 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:54,960 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:31:54,960 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:31:54,962 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:31:54,962 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:31:54,963 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:54,963 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:54,964 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:31:54,964 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:54,965 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:54,966 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,38255,1696249782114': 2023-10-02T12:31:54,966 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:31:54,966 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:31:54,967 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:31:54,967 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:31:54,967 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:31:54,967 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:31:54,970 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:31:54,970 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:31:54,970 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:31:54,970 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:31:54,970 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:31:54,970 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:31:54,970 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:31:54,971 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:31:54,971 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:31:54,971 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:54,972 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:31:54,972 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:54,973 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:54,973 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:31:54,973 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:31:54,980 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:31:54,986 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:31:54,986 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:31:54,986 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:31:54,986 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:31:54,986 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:31:54,986 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:31:54,986 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:31:55,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:31:55,091 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249915090_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } 2023-10-02T12:31:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249915091 (current time:1696249915091). 2023-10-02T12:31:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:31:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249915090_ns2_test-16962497916551 VERSION not specified, setting to 2 2023-10-02T12:31:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a679865, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:55,099 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:32780, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:55,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:31:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=135, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1696249915090_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } 2023-10-02T12:31:55,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249915090_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 }, snapshot procedure id = 135 2023-10-02T12:31:55,106 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1696249915090_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:31:55,107 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1696249915090_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:31:55,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=135 2023-10-02T12:31:55,110 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249915090_ns2_test-16962497916551 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor129.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:31:55,114 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1696249915090_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:31:55,130 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1696249915090_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:31:55,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE; SnapshotRegionProcedure 1ff8cee0e4d9c23004d17b9d828c2c96}] 2023-10-02T12:31:55,133 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=136, ppid=135, state=RUNNABLE; SnapshotRegionProcedure 1ff8cee0e4d9c23004d17b9d828c2c96 2023-10-02T12:31:55,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=135 2023-10-02T12:31:55,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=136 2023-10-02T12:31:55,284 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96. 2023-10-02T12:31:55,284 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.HRegion(2520): Flush status journal for 1ff8cee0e4d9c23004d17b9d828c2c96: 2023-10-02T12:31:55,284 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96. for snapshot_1696249915090_ns2_test-16962497916551 completed. 2023-10-02T12:31:55,284 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96.' region-info for snapshot=snapshot_1696249915090_ns2_test-16962497916551 2023-10-02T12:31:55,285 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:31:55,285 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96/f/eec10f547780452b96a08ad7a8ce56ca_SeqId_4_] hfiles 2023-10-02T12:31:55,285 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96/f/eec10f547780452b96a08ad7a8ce56ca_SeqId_4_ for snapshot=snapshot_1696249915090_ns2_test-16962497916551 2023-10-02T12:31:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=135 2023-10-02T12:31:55,699 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96. 2023-10-02T12:31:55,699 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=136 2023-10-02T12:31:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=136 2023-10-02T12:31:55,700 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249915090_ns2_test-16962497916551 on region 1ff8cee0e4d9c23004d17b9d828c2c96 2023-10-02T12:31:55,702 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=136, ppid=135, state=RUNNABLE; SnapshotRegionProcedure 1ff8cee0e4d9c23004d17b9d828c2c96 2023-10-02T12:31:55,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=136, resume processing ppid=135 2023-10-02T12:31:55,705 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1696249915090_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:31:55,706 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1696249915090_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:31:55,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=136, ppid=135, state=SUCCESS; SnapshotRegionProcedure 1ff8cee0e4d9c23004d17b9d828c2c96 in 572 msec 2023-10-02T12:31:55,706 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1696249915090_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:31:55,706 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249915090_ns2_test-16962497916551 2023-10-02T12:31:55,707 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249915090_ns2_test-16962497916551 2023-10-02T12:31:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=135 2023-10-02T12:31:55,740 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1696249915090_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:31:55,746 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1696249915090_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:31:55,747 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249915090_ns2_test-16962497916551 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249915090_ns2_test-16962497916551 2023-10-02T12:31:55,750 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1696249915090_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:31:55,750 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249915090_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 }, snapshot procedure id = 135 2023-10-02T12:31:55,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=135, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1696249915090_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } in 648 msec 2023-10-02T12:31:55,864 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:31:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=135 2023-10-02T12:31:56,215 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249916214_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } 2023-10-02T12:31:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249916215 (current time:1696249916215). 2023-10-02T12:31:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:31:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249916214_ns3_test-16962497916552 VERSION not specified, setting to 2 2023-10-02T12:31:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36c9e35c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:31:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:31:56,226 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:32782, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:31:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:31:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:31:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=137, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1696249916214_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } 2023-10-02T12:31:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249916214_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 }, snapshot procedure id = 137 2023-10-02T12:31:56,231 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1696249916214_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:31:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=137 2023-10-02T12:31:56,238 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1696249916214_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:31:56,242 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249916214_ns3_test-16962497916552 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor129.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:31:56,245 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1696249916214_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:31:56,257 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1696249916214_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:31:56,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE; SnapshotRegionProcedure 8f9314653c5767b76ceff134e893c29a}] 2023-10-02T12:31:56,259 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=138, ppid=137, state=RUNNABLE; SnapshotRegionProcedure 8f9314653c5767b76ceff134e893c29a 2023-10-02T12:31:56,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=137 2023-10-02T12:31:56,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=138 2023-10-02T12:31:56,411 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a. 2023-10-02T12:31:56,413 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.HRegion(2520): Flush status journal for 8f9314653c5767b76ceff134e893c29a: 2023-10-02T12:31:56,413 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a. for snapshot_1696249916214_ns3_test-16962497916552 completed. 2023-10-02T12:31:56,413 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a.' region-info for snapshot=snapshot_1696249916214_ns3_test-16962497916552 2023-10-02T12:31:56,413 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:31:56,413 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:31:56,420 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a. 2023-10-02T12:31:56,420 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=138 2023-10-02T12:31:56,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=138 2023-10-02T12:31:56,420 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249916214_ns3_test-16962497916552 on region 8f9314653c5767b76ceff134e893c29a 2023-10-02T12:31:56,422 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=138, ppid=137, state=RUNNABLE; SnapshotRegionProcedure 8f9314653c5767b76ceff134e893c29a 2023-10-02T12:31:56,424 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=138, resume processing ppid=137 2023-10-02T12:31:56,424 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=138, ppid=137, state=SUCCESS; SnapshotRegionProcedure 8f9314653c5767b76ceff134e893c29a in 166 msec 2023-10-02T12:31:56,424 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1696249916214_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:31:56,425 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1696249916214_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:31:56,425 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1696249916214_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:31:56,425 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249916214_ns3_test-16962497916552 2023-10-02T12:31:56,427 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249916214_ns3_test-16962497916552 2023-10-02T12:31:56,438 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1696249916214_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:31:56,443 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1696249916214_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:31:56,444 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249916214_ns3_test-16962497916552 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249916214_ns3_test-16962497916552 2023-10-02T12:31:56,445 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1696249916214_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:31:56,445 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249916214_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 }, snapshot procedure id = 137 2023-10-02T12:31:56,447 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=137, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1696249916214_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } in 217 msec 2023-10-02T12:31:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=137 2023-10-02T12:31:56,536 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696249912589 2023-10-02T12:31:56,536 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:31:56,537 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-10-02T12:31:56,537 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249912589_ns2:test-16962497916551 2023-10-02T12:31:56,537 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249915090_ns2_test-16962497916551 to hdfs://localhost:37823/backupUT/backup_1696249912589/ns2/test-16962497916551/ 2023-10-02T12:31:56,537 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:31:56,573 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:31:56,573 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,40619,1696249778965_528182803_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37823/backupUT/backup_1696249912589/ns2/test-16962497916551, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37823/backupUT/backup_1696249912589/ns2/test-16962497916551/.hbase-snapshot/.tmp/snapshot_1696249915090_ns2_test-16962497916551 2023-10-02T12:31:56,573 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07, snapshotDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249915090_ns2_test-16962497916551. 2023-10-02T12:31:56,584 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249915090_ns2_test-16962497916551 to hdfs://localhost:37823/backupUT/backup_1696249912589/ns2/test-16962497916551/.hbase-snapshot/.tmp/snapshot_1696249915090_ns2_test-16962497916551 2023-10-02T12:31:56,607 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:56,607 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:56,608 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:56,608 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:56,980 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-16962497916552' 2023-10-02T12:31:57,812 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-8921736372871734932.jar 2023-10-02T12:31:57,813 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:57,813 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:57,814 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:57,814 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:57,814 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:57,815 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:57,815 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:57,815 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:57,816 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:57,816 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:31:57,817 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:31:57,817 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:31:57,818 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:31:57,818 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:31:57,819 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:31:57,819 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:31:57,819 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:31:57,820 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:31:57,820 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:31:57,820 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:31:57,821 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:31:57,821 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:57,822 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:57,822 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:57,823 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:57,823 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:31:57,823 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:57,824 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:31:57,916 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249915090_ns2_test-16962497916551' hfile list 2023-10-02T12:31:57,921 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-10-02T12:31:58,114 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249918053/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:31:58,114 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249918053/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:31:58,195 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:31:58,220 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96/f/eec10f547780452b96a08ad7a8ce56ca_SeqId_4_ output=hdfs://localhost:37823/backupUT/backup_1696249912589/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96/f/eec10f547780452b96a08ad7a8ce56ca_SeqId_4_ 2023-10-02T12:31:58,220 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 8.321M/sec 2023-10-02T12:31:59,168 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:31:59,169 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:31:59,176 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249915090_ns2_test-16962497916551 2023-10-02T12:31:59,176 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249915090_ns2_test-16962497916551 finished. 2023-10-02T12:31:59,176 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249912589_ns3:test-16962497916552 2023-10-02T12:31:59,176 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249916214_ns3_test-16962497916552 to hdfs://localhost:37823/backupUT/backup_1696249912589/ns3/test-16962497916552/ 2023-10-02T12:31:59,177 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:31:59,213 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:31:59,213 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,40619,1696249778965_-243339218_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37823/backupUT/backup_1696249912589/ns3/test-16962497916552, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37823/backupUT/backup_1696249912589/ns3/test-16962497916552/.hbase-snapshot/.tmp/snapshot_1696249916214_ns3_test-16962497916552 2023-10-02T12:31:59,213 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07, snapshotDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249916214_ns3_test-16962497916552. 2023-10-02T12:31:59,224 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249916214_ns3_test-16962497916552 to hdfs://localhost:37823/backupUT/backup_1696249912589/ns3/test-16962497916552/.hbase-snapshot/.tmp/snapshot_1696249916214_ns3_test-16962497916552 2023-10-02T12:31:59,657 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:59,658 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:59,659 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:31:59,659 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:00,895 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-1100567468389709393.jar 2023-10-02T12:32:00,896 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:00,896 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:00,896 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:00,897 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:00,897 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:00,898 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:00,898 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:00,898 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:00,899 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:00,899 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:32:00,900 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:32:00,900 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:32:00,901 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:32:00,901 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:32:00,902 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:32:00,902 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:32:00,902 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:32:00,902 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:32:00,903 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:32:00,903 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:32:00,904 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:32:00,904 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:32:00,905 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:32:00,905 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:32:00,906 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:32:00,906 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:32:00,906 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:32:00,907 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:32:01,006 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249916214_ns3_test-16962497916552' hfile list 2023-10-02T12:32:01,104 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249921059/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:32:01,104 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249921059/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:32:02,136 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:32:02,137 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:32:02,142 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249916214_ns3_test-16962497916552 2023-10-02T12:32:02,142 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249916214_ns3_test-16962497916552 finished. 2023-10-02T12:32:02,142 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1034): ns2:test-16962497916551 2023-10-02T12:32:02,142 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1034): ns3:test-16962497916552 2023-10-02T12:32:02,150 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249912589 2023-10-02T12:32:02,150 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:32:02,160 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37823/backupUT/backup_1696249912589/.backup.manifest 2023-10-02T12:32:02,160 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249912589 2023-10-02T12:32:02,160 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:32:02,570 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37823/backupUT/backup_1696249912589/.backup.manifest 2023-10-02T12:32:02,570 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(358): Backup backup_1696249912589 finished: type=FULL,tablelist=ns2:test-16962497916551;ns3:test-16962497916552,targetRootDir=hdfs://localhost:37823/backupUT,startts=1696249914840,completets=1696249922150,bytescopied=0 2023-10-02T12:32:02,570 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:32:02,570 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249915090_ns2_test-16962497916551 2023-10-02T12:32:02,571 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249915090_ns2_test-16962497916551" 2023-10-02T12:32:02,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249915090_ns2_test-16962497916551 2023-10-02T12:32:02,577 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249915090_ns2_test-16962497916551 for backup backup_1696249912589 succeeded. 2023-10-02T12:32:02,578 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249916214_ns3_test-16962497916552 2023-10-02T12:32:02,579 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249916214_ns3_test-16962497916552" 2023-10-02T12:32:02,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249916214_ns3_test-16962497916552 2023-10-02T12:32:02,582 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249916214_ns3_test-16962497916552 for backup backup_1696249912589 succeeded. 2023-10-02T12:32:02,584 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:32:02,592 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:32:02,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:32:02,596 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:32:02,603 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:32:02,610 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(376): Backup backup_1696249912589 completed. 2023-10-02T12:32:02,610 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:02,639 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bc4cc31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:02,643 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:02,645 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49014, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:02,646 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(101): HBase table ns2:table2 does not exist. It will be created during restore process 2023-10-02T12:32:02,647 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(101): HBase table ns3:table3_restore does not exist. It will be created during restore process 2023-10-02T12:32:02,648 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249912589/.backup.manifest 2023-10-02T12:32:02,650 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249912589/.backup.manifest 2023-10-02T12:32:02,651 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249912589/.backup.manifest 2023-10-02T12:32:02,653 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249912589/.backup.manifest 2023-10-02T12:32:02,654 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249912589/.backup.manifest 2023-10-02T12:32:02,656 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249912589/.backup.manifest 2023-10-02T12:32:02,656 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-16962497916551' to 'ns2:table2' from full backup image hdfs://localhost:37823/backupUT/backup_1696249912589/ns2/test-16962497916551 2023-10-02T12:32:02,666 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(487): Creating target table 'ns2:table2' 2023-10-02T12:32:02,666 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:37823/backupUT/backup_1696249912589/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96 2023-10-02T12:32:02,667 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:37823/backupUT/backup_1696249912589/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96/f in region [hdfs://localhost:37823/backupUT/backup_1696249912589/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96] 2023-10-02T12:32:02,671 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:37823/backupUT/backup_1696249912589/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96/f/eec10f547780452b96a08ad7a8ce56ca_SeqId_4_ first=row0 last=row98 2023-10-02T12:32:02,672 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:32:02,673 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51346, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:32:02,674 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:32:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=139, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:table2 2023-10-02T12:32:02,678 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:32:02,678 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns2" qualifier: "table2" procId is: 139 2023-10-02T12:32:02,679 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:32:02,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=139 2023-10-02T12:32:02,686 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 7dcb074b404fab5e5f2e64a2123c81c4, NAME => 'ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:32:02,695 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:32:02,695 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1668): Closing 7dcb074b404fab5e5f2e64a2123c81c4, disabling compactions & flushes 2023-10-02T12:32:02,695 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4. 2023-10-02T12:32:02,695 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4. 2023-10-02T12:32:02,695 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4. after waiting 0 ms 2023-10-02T12:32:02,695 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4. 2023-10-02T12:32:02,695 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1909): Closed ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4. 2023-10-02T12:32:02,695 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1622): Region close journal for 7dcb074b404fab5e5f2e64a2123c81c4: 2023-10-02T12:32:02,696 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:32:02,696 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1696249922696"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249922696"}]},"ts":"1696249922696"} 2023-10-02T12:32:02,698 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:32:02,699 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:32:02,699 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249922699"}]},"ts":"1696249922699"} 2023-10-02T12:32:02,700 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2023-10-02T12:32:02,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=7dcb074b404fab5e5f2e64a2123c81c4, ASSIGN}] 2023-10-02T12:32:02,715 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=7dcb074b404fab5e5f2e64a2123c81c4, ASSIGN 2023-10-02T12:32:02,716 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:table2, region=7dcb074b404fab5e5f2e64a2123c81c4, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38255,1696249782114; forceNewPlan=false, retain=false 2023-10-02T12:32:02,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=139 2023-10-02T12:32:02,868 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=140 updating hbase:meta row=7dcb074b404fab5e5f2e64a2123c81c4, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:02,869 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE; OpenRegionProcedure 7dcb074b404fab5e5f2e64a2123c81c4, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:32:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=139 2023-10-02T12:32:03,027 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(130): Open ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4. 2023-10-02T12:32:03,027 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7254): Opening region: {ENCODED => 7dcb074b404fab5e5f2e64a2123c81c4, NAME => 'ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:32:03,027 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:32:03,027 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 7dcb074b404fab5e5f2e64a2123c81c4 2023-10-02T12:32:03,028 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(888): Instantiated ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:32:03,028 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7296): checking encryption for 7dcb074b404fab5e5f2e64a2123c81c4 2023-10-02T12:32:03,028 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7299): checking classloading for 7dcb074b404fab5e5f2e64a2123c81c4 2023-10-02T12:32:03,031 INFO [StoreOpener-7dcb074b404fab5e5f2e64a2123c81c4-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 7dcb074b404fab5e5f2e64a2123c81c4 2023-10-02T12:32:03,043 INFO [StoreOpener-7dcb074b404fab5e5f2e64a2123c81c4-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7dcb074b404fab5e5f2e64a2123c81c4 columnFamilyName f 2023-10-02T12:32:03,043 DEBUG [StoreOpener-7dcb074b404fab5e5f2e64a2123c81c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:32:03,044 INFO [StoreOpener-7dcb074b404fab5e5f2e64a2123c81c4-1 {}] regionserver.HStore(324): Store=7dcb074b404fab5e5f2e64a2123c81c4/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:32:03,045 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/7dcb074b404fab5e5f2e64a2123c81c4 2023-10-02T12:32:03,045 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/7dcb074b404fab5e5f2e64a2123c81c4 2023-10-02T12:32:03,048 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1076): writing seq id for 7dcb074b404fab5e5f2e64a2123c81c4 2023-10-02T12:32:03,052 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/7dcb074b404fab5e5f2e64a2123c81c4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:32:03,052 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1093): Opened 7dcb074b404fab5e5f2e64a2123c81c4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11521895520, jitterRate=0.07306014001369476}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:32:03,053 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(992): Region open journal for 7dcb074b404fab5e5f2e64a2123c81c4: 2023-10-02T12:32:03,054 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4., pid=141, masterSystemTime=1696249923021 2023-10-02T12:32:03,056 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4. 2023-10-02T12:32:03,056 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(158): Opened ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4. 2023-10-02T12:32:03,056 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=140 updating hbase:meta row=7dcb074b404fab5e5f2e64a2123c81c4, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:03,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=141, resume processing ppid=140 2023-10-02T12:32:03,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=141, ppid=140, state=SUCCESS; OpenRegionProcedure 7dcb074b404fab5e5f2e64a2123c81c4, server=jenkins-hbase3.apache.org,38255,1696249782114 in 190 msec 2023-10-02T12:32:03,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=140, resume processing ppid=139 2023-10-02T12:32:03,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=140, ppid=139, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=7dcb074b404fab5e5f2e64a2123c81c4, ASSIGN in 347 msec 2023-10-02T12:32:03,063 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:32:03,063 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249923063"}]},"ts":"1696249923063"} 2023-10-02T12:32:03,064 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2023-10-02T12:32:03,068 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:32:03,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=139, state=SUCCESS; CreateTableProcedure table=ns2:table2 in 394 msec 2023-10-02T12:32:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=139 2023-10-02T12:32:03,291 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:37823/backupUT/backup_1696249912589/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96 from hbase tables ns2:test-16962497916551 to tables ns2:table2 2023-10-02T12:32:03,291 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-16962497916551 into ns2:table2 2023-10-02T12:32:03,293 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-table2-1696249923291 from hdfs://localhost:37823/backupUT/backup_1696249912589/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96 2023-10-02T12:32:03,299 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@91b65d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:03,300 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:32:03,301 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51348, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:32:03,302 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-10-02T12:32:03,302 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:table2 2023-10-02T12:32:03,307 DEBUG [hconnection-0x61a3cb0a-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:03,308 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49016, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:03,310 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:table2: 2023-10-02T12:32:03,310 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-10-02T12:32:03,310 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/partitions_25c50c39-8dc0-42c9-abc9-9d37fe6d6b21 2023-10-02T12:32:03,721 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:03,722 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:03,722 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:03,722 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:04,761 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-3174402622556187648.jar 2023-10-02T12:32:04,762 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:04,763 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:04,763 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:04,764 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:04,764 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:04,765 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:04,765 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:04,765 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:04,766 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:04,767 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:32:04,768 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:32:04,768 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:32:04,769 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:32:04,769 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:32:04,769 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:32:04,770 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:32:04,770 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:32:04,771 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:32:04,771 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:32:04,772 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:32:04,772 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:32:04,773 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:04,773 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:04,773 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:04,774 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:04,774 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:04,774 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:04,775 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:32:04,775 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:table2 2023-10-02T12:32:04,776 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:04,776 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-10-02T12:32:04,776 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-10-02T12:32:04,777 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:32:04,794 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-10-02T12:32:04,881 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249924838/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:32:04,881 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249924838/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:32:04,936 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:37823/backupUT/backup_1696249912589/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96/f/eec10f547780452b96a08ad7a8ce56ca_SeqId_4_ 2023-10-02T12:32:04,941 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-10-02T12:32:04,966 DEBUG [pool-484-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39bcd59b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:04,967 DEBUG [hconnection-0x7494d496-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:04,968 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49020, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:04,975 DEBUG [pool-484-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:04,975 DEBUG [pool-484-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-10-02T12:32:04,975 DEBUG [pool-484-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-10-02T12:32:04,980 DEBUG [pool-484-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-table2-1696249923291/_temporary/0/_temporary/attempt_local533812793_0022_r_000000_0/f/9975e33a69ba442eb67dd5904fd90388 is 37, key is row10/f:q1/1696249795545/Put/seqid=0 2023-10-02T12:32:04,982 WARN [Thread-2112 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:38255] Specified favored nodes: [jenkins-hbase3.apache.org:38255] 2023-10-02T12:32:05,930 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-table2-1696249923291 2023-10-02T12:32:05,934 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514ad250, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:05,936 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:05,937 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49036, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:05,943 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-table2-1696249923291/_SUCCESS 2023-10-02T12:32:05,947 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:32:05,948 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51358, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:32:05,959 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-table2-1696249923291/f/9975e33a69ba442eb67dd5904fd90388 first=Optional[row0] last=Optional[row98] 2023-10-02T12:32:05,960 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4., hostname=jenkins-hbase3.apache.org,38255,1696249782114, seqNum=2 for row with hfile group [{f,hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-table2-1696249923291/f/9975e33a69ba442eb67dd5904fd90388}] 2023-10-02T12:32:05,965 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-table2-1696249923291/f/9975e33a69ba442eb67dd5904fd90388 for inclusion in 7dcb074b404fab5e5f2e64a2123c81c4/f 2023-10-02T12:32:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-10-02T12:32:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(625): Region bounds: first= last= 2023-10-02T12:32:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HRegion(2520): Flush status journal for 7dcb074b404fab5e5f2e64a2123c81c4: 2023-10-02T12:32:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-ns2-table2-1696249923291/f/9975e33a69ba442eb67dd5904fd90388 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__ns2_table2__llcfc7iem2mksh5pkn7oej3ltm49llt1ot293to3onfv8dk7peblopp2vgjtv4fu/f/9975e33a69ba442eb67dd5904fd90388 2023-10-02T12:32:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__ns2_table2__llcfc7iem2mksh5pkn7oej3ltm49llt1ot293to3onfv8dk7peblopp2vgjtv4fu/f/9975e33a69ba442eb67dd5904fd90388 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/7dcb074b404fab5e5f2e64a2123c81c4/f/f2ad68fa59bb4515a64841cc5d861880_SeqId_4_ 2023-10-02T12:32:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dabfb4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:32:05,979 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51368, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:32:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:05,983 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49042, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:06,001 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__ns2_table2__llcfc7iem2mksh5pkn7oej3ltm49llt1ot293to3onfv8dk7peblopp2vgjtv4fu/f/9975e33a69ba442eb67dd5904fd90388 into 7dcb074b404fab5e5f2e64a2123c81c4/f as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/7dcb074b404fab5e5f2e64a2123c81c4/f/f2ad68fa59bb4515a64841cc5d861880_SeqId_4_ - updating store file list. 2023-10-02T12:32:06,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/7dcb074b404fab5e5f2e64a2123c81c4/f/f2ad68fa59bb4515a64841cc5d861880_SeqId_4_ into 7dcb074b404fab5e5f2e64a2123c81c4/f 2023-10-02T12:32:06,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__ns2_table2__llcfc7iem2mksh5pkn7oej3ltm49llt1ot293to3onfv8dk7peblopp2vgjtv4fu/f/9975e33a69ba442eb67dd5904fd90388 into 7dcb074b404fab5e5f2e64a2123c81c4/f (new location: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/7dcb074b404fab5e5f2e64a2123c81c4/f/f2ad68fa59bb4515a64841cc5d861880_SeqId_4_) 2023-10-02T12:32:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__ns2_table2__llcfc7iem2mksh5pkn7oej3ltm49llt1ot293to3onfv8dk7peblopp2vgjtv4fu/f/9975e33a69ba442eb67dd5904fd90388 2023-10-02T12:32:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f51da35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:32:06,104 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51384, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-10-02T12:32:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:06,110 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49044, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-10-02T12:32:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:06,135 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:06,135 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-10-02T12:32:06,135 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-10-02T12:32:06,135 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(233): Backup: backup_1696249912589 hdfs://localhost:37823/backupUT/backup_1696249912589/ns2/test-16962497916551/ 2023-10-02T12:32:06,136 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249912589/.backup.manifest 2023-10-02T12:32:06,139 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249912589/.backup.manifest 2023-10-02T12:32:06,139 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-16962497916552' to 'ns3:table3_restore' from full backup image hdfs://localhost:37823/backupUT/backup_1696249912589/ns3/test-16962497916552 2023-10-02T12:32:06,144 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:37823/backupUT/backup_1696249912589/ns3/test-16962497916552/archive/data/ns3/test-16962497916552 does not exists 2023-10-02T12:32:06,144 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-16962497916552, will only create table 2023-10-02T12:32:06,145 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(487): Creating target table 'ns3:table3_restore' 2023-10-02T12:32:06,146 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:32:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:table3_restore 2023-10-02T12:32:06,148 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:32:06,148 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "ns3" qualifier: "table3_restore" procId is: 142 2023-10-02T12:32:06,149 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:32:06,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-10-02T12:32:06,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-10-02T12:32:06,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-10-02T12:32:06,556 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 504ca2b90001e75c082d0a09cb1507e6, NAME => 'ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:32:06,562 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:32:06,562 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1668): Closing 504ca2b90001e75c082d0a09cb1507e6, disabling compactions & flushes 2023-10-02T12:32:06,562 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6. 2023-10-02T12:32:06,562 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6. 2023-10-02T12:32:06,562 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6. after waiting 0 ms 2023-10-02T12:32:06,562 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6. 2023-10-02T12:32:06,562 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6. 2023-10-02T12:32:06,562 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1622): Region close journal for 504ca2b90001e75c082d0a09cb1507e6: 2023-10-02T12:32:06,563 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:32:06,563 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1696249926563"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249926563"}]},"ts":"1696249926563"} 2023-10-02T12:32:06,565 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:32:06,566 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:32:06,566 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249926566"}]},"ts":"1696249926566"} 2023-10-02T12:32:06,567 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLING in hbase:meta 2023-10-02T12:32:06,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=504ca2b90001e75c082d0a09cb1507e6, ASSIGN}] 2023-10-02T12:32:06,575 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=504ca2b90001e75c082d0a09cb1507e6, ASSIGN 2023-10-02T12:32:06,576 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:table3_restore, region=504ca2b90001e75c082d0a09cb1507e6, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38255,1696249782114; forceNewPlan=false, retain=false 2023-10-02T12:32:06,728 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=143 updating hbase:meta row=504ca2b90001e75c082d0a09cb1507e6, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:06,730 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE; OpenRegionProcedure 504ca2b90001e75c082d0a09cb1507e6, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:32:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-10-02T12:32:06,910 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(130): Open ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6. 2023-10-02T12:32:06,910 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7254): Opening region: {ENCODED => 504ca2b90001e75c082d0a09cb1507e6, NAME => 'ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:32:06,910 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:32:06,911 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table3_restore 504ca2b90001e75c082d0a09cb1507e6 2023-10-02T12:32:06,911 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:32:06,911 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7296): checking encryption for 504ca2b90001e75c082d0a09cb1507e6 2023-10-02T12:32:06,911 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7299): checking classloading for 504ca2b90001e75c082d0a09cb1507e6 2023-10-02T12:32:06,916 INFO [StoreOpener-504ca2b90001e75c082d0a09cb1507e6-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 504ca2b90001e75c082d0a09cb1507e6 2023-10-02T12:32:06,917 INFO [StoreOpener-504ca2b90001e75c082d0a09cb1507e6-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 504ca2b90001e75c082d0a09cb1507e6 columnFamilyName f 2023-10-02T12:32:06,917 DEBUG [StoreOpener-504ca2b90001e75c082d0a09cb1507e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:32:06,918 INFO [StoreOpener-504ca2b90001e75c082d0a09cb1507e6-1 {}] regionserver.HStore(324): Store=504ca2b90001e75c082d0a09cb1507e6/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:32:06,919 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/table3_restore/504ca2b90001e75c082d0a09cb1507e6 2023-10-02T12:32:06,919 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/table3_restore/504ca2b90001e75c082d0a09cb1507e6 2023-10-02T12:32:06,921 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1076): writing seq id for 504ca2b90001e75c082d0a09cb1507e6 2023-10-02T12:32:06,923 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/table3_restore/504ca2b90001e75c082d0a09cb1507e6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:32:06,923 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): Opened 504ca2b90001e75c082d0a09cb1507e6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10818822560, jitterRate=0.007581368088722229}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:32:06,924 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(992): Region open journal for 504ca2b90001e75c082d0a09cb1507e6: 2023-10-02T12:32:06,924 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6., pid=144, masterSystemTime=1696249926882 2023-10-02T12:32:06,926 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6. 2023-10-02T12:32:06,926 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(158): Opened ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6. 2023-10-02T12:32:06,926 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=143 updating hbase:meta row=504ca2b90001e75c082d0a09cb1507e6, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:06,929 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=144, resume processing ppid=143 2023-10-02T12:32:06,929 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=144, ppid=143, state=SUCCESS; OpenRegionProcedure 504ca2b90001e75c082d0a09cb1507e6, server=jenkins-hbase3.apache.org,38255,1696249782114 in 197 msec 2023-10-02T12:32:06,931 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=143, resume processing ppid=142 2023-10-02T12:32:06,931 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=143, ppid=142, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=504ca2b90001e75c082d0a09cb1507e6, ASSIGN in 356 msec 2023-10-02T12:32:06,931 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:32:06,932 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249926931"}]},"ts":"1696249926931"} 2023-10-02T12:32:06,933 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLED in hbase:meta 2023-10-02T12:32:06,935 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:32:06,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=142, state=SUCCESS; CreateTableProcedure table=ns3:table3_restore in 789 msec 2023-10-02T12:32:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=142 2023-10-02T12:32:07,258 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-10-02T12:32:07,258 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(233): Backup: backup_1696249912589 hdfs://localhost:37823/backupUT/backup_1696249912589/ns3/test-16962497916552/ 2023-10-02T12:32:07,258 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-10-02T12:32:07,258 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:07,268 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable ns2:table2 2023-10-02T12:32:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=145, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:table2 2023-10-02T12:32:07,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=145 2023-10-02T12:32:07,271 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249927271"}]},"ts":"1696249927271"} 2023-10-02T12:32:07,273 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLING in hbase:meta 2023-10-02T12:32:07,274 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set ns2:table2 to state=DISABLING 2023-10-02T12:32:07,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=7dcb074b404fab5e5f2e64a2123c81c4, UNASSIGN}] 2023-10-02T12:32:07,277 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=146, ppid=145, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=7dcb074b404fab5e5f2e64a2123c81c4, UNASSIGN 2023-10-02T12:32:07,277 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=146 updating hbase:meta row=7dcb074b404fab5e5f2e64a2123c81c4, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:07,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE; CloseRegionProcedure 7dcb074b404fab5e5f2e64a2123c81c4, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:32:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=145 2023-10-02T12:32:07,430 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] handler.UnassignRegionHandler(119): Close 7dcb074b404fab5e5f2e64a2123c81c4 2023-10-02T12:32:07,430 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1668): Closing 7dcb074b404fab5e5f2e64a2123c81c4, disabling compactions & flushes 2023-10-02T12:32:07,431 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1690): Closing region ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4. 2023-10-02T12:32:07,431 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4. 2023-10-02T12:32:07,431 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4. after waiting 0 ms 2023-10-02T12:32:07,431 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4. 2023-10-02T12:32:07,438 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/7dcb074b404fab5e5f2e64a2123c81c4/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-10-02T12:32:07,439 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:32:07,439 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1909): Closed ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4. 2023-10-02T12:32:07,439 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1622): Region close journal for 7dcb074b404fab5e5f2e64a2123c81c4: 2023-10-02T12:32:07,441 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] handler.UnassignRegionHandler(163): Closed 7dcb074b404fab5e5f2e64a2123c81c4 2023-10-02T12:32:07,441 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=146 updating hbase:meta row=7dcb074b404fab5e5f2e64a2123c81c4, regionState=CLOSED 2023-10-02T12:32:07,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=147, resume processing ppid=146 2023-10-02T12:32:07,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=147, ppid=146, state=SUCCESS; CloseRegionProcedure 7dcb074b404fab5e5f2e64a2123c81c4, server=jenkins-hbase3.apache.org,38255,1696249782114 in 164 msec 2023-10-02T12:32:07,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=146, resume processing ppid=145 2023-10-02T12:32:07,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=146, ppid=145, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=7dcb074b404fab5e5f2e64a2123c81c4, UNASSIGN in 169 msec 2023-10-02T12:32:07,446 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249927446"}]},"ts":"1696249927446"} 2023-10-02T12:32:07,447 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLED in hbase:meta 2023-10-02T12:32:07,450 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set ns2:table2 to state=DISABLED 2023-10-02T12:32:07,451 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=145, state=SUCCESS; DisableTableProcedure table=ns2:table2 in 183 msec 2023-10-02T12:32:07,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=145 2023-10-02T12:32:07,574 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete ns2:table2 2023-10-02T12:32:07,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=148, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns2:table2 2023-10-02T12:32:07,577 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=148, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-10-02T12:32:07,578 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=148, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns2:table2 2023-10-02T12:32:07,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=148 2023-10-02T12:32:07,581 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/7dcb074b404fab5e5f2e64a2123c81c4 2023-10-02T12:32:07,582 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/7dcb074b404fab5e5f2e64a2123c81c4/f, FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/7dcb074b404fab5e5f2e64a2123c81c4/recovered.edits] 2023-10-02T12:32:07,586 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/7dcb074b404fab5e5f2e64a2123c81c4/f/f2ad68fa59bb4515a64841cc5d861880_SeqId_4_ to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/archive/data/ns2/table2/7dcb074b404fab5e5f2e64a2123c81c4/f/f2ad68fa59bb4515a64841cc5d861880_SeqId_4_ 2023-10-02T12:32:07,589 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/7dcb074b404fab5e5f2e64a2123c81c4/recovered.edits/6.seqid to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/archive/data/ns2/table2/7dcb074b404fab5e5f2e64a2123c81c4/recovered.edits/6.seqid 2023-10-02T12:32:07,592 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/table2/7dcb074b404fab5e5f2e64a2123c81c4 2023-10-02T12:32:07,592 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived ns2:table2 regions 2023-10-02T12:32:07,594 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=148, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns2:table2 2023-10-02T12:32:07,596 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:table2 from hbase:meta 2023-10-02T12:32:07,598 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:table2' descriptor. 2023-10-02T12:32:07,599 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=148, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns2:table2 2023-10-02T12:32:07,599 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:table2' from region states. 2023-10-02T12:32:07,600 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696249927599"}]},"ts":"9223372036854775807"} 2023-10-02T12:32:07,601 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:32:07,601 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 7dcb074b404fab5e5f2e64a2123c81c4, NAME => 'ns2:table2,,1696249922674.7dcb074b404fab5e5f2e64a2123c81c4.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:32:07,601 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:table2' as deleted. 2023-10-02T12:32:07,602 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1696249927601"}]},"ts":"9223372036854775807"} 2023-10-02T12:32:07,605 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:table2 state from META 2023-10-02T12:32:07,607 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(127): Finished pid=148, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-10-02T12:32:07,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=148, state=SUCCESS; DeleteTableProcedure table=ns2:table2 in 33 msec 2023-10-02T12:32:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=148 2023-10-02T12:32:07,680 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable ns3:table3_restore 2023-10-02T12:32:07,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=149, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:table3_restore 2023-10-02T12:32:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=149 2023-10-02T12:32:07,684 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249927683"}]},"ts":"1696249927683"} 2023-10-02T12:32:07,685 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLING in hbase:meta 2023-10-02T12:32:07,686 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set ns3:table3_restore to state=DISABLING 2023-10-02T12:32:07,687 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=504ca2b90001e75c082d0a09cb1507e6, UNASSIGN}] 2023-10-02T12:32:07,688 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=150, ppid=149, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=504ca2b90001e75c082d0a09cb1507e6, UNASSIGN 2023-10-02T12:32:07,689 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=150 updating hbase:meta row=504ca2b90001e75c082d0a09cb1507e6, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:07,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE; CloseRegionProcedure 504ca2b90001e75c082d0a09cb1507e6, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:32:07,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=149 2023-10-02T12:32:07,842 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] handler.UnassignRegionHandler(119): Close 504ca2b90001e75c082d0a09cb1507e6 2023-10-02T12:32:07,843 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1668): Closing 504ca2b90001e75c082d0a09cb1507e6, disabling compactions & flushes 2023-10-02T12:32:07,843 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6. 2023-10-02T12:32:07,843 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6. 2023-10-02T12:32:07,843 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6. after waiting 0 ms 2023-10-02T12:32:07,843 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6. 2023-10-02T12:32:07,847 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/table3_restore/504ca2b90001e75c082d0a09cb1507e6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-10-02T12:32:07,848 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:32:07,848 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6. 2023-10-02T12:32:07,848 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1622): Region close journal for 504ca2b90001e75c082d0a09cb1507e6: 2023-10-02T12:32:07,850 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] handler.UnassignRegionHandler(163): Closed 504ca2b90001e75c082d0a09cb1507e6 2023-10-02T12:32:07,850 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=150 updating hbase:meta row=504ca2b90001e75c082d0a09cb1507e6, regionState=CLOSED 2023-10-02T12:32:07,852 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=151, resume processing ppid=150 2023-10-02T12:32:07,852 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=151, ppid=150, state=SUCCESS; CloseRegionProcedure 504ca2b90001e75c082d0a09cb1507e6, server=jenkins-hbase3.apache.org,38255,1696249782114 in 161 msec 2023-10-02T12:32:07,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=150, resume processing ppid=149 2023-10-02T12:32:07,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=150, ppid=149, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=504ca2b90001e75c082d0a09cb1507e6, UNASSIGN in 165 msec 2023-10-02T12:32:07,854 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249927854"}]},"ts":"1696249927854"} 2023-10-02T12:32:07,855 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLED in hbase:meta 2023-10-02T12:32:07,857 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set ns3:table3_restore to state=DISABLED 2023-10-02T12:32:07,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=149, state=SUCCESS; DisableTableProcedure table=ns3:table3_restore in 178 msec 2023-10-02T12:32:07,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=149 2023-10-02T12:32:07,987 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete ns3:table3_restore 2023-10-02T12:32:07,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=152, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns3:table3_restore 2023-10-02T12:32:07,990 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=152, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-10-02T12:32:07,990 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=152, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-10-02T12:32:07,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=152 2023-10-02T12:32:07,992 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/table3_restore/504ca2b90001e75c082d0a09cb1507e6 2023-10-02T12:32:07,994 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/table3_restore/504ca2b90001e75c082d0a09cb1507e6/f, FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/table3_restore/504ca2b90001e75c082d0a09cb1507e6/recovered.edits] 2023-10-02T12:32:07,999 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/table3_restore/504ca2b90001e75c082d0a09cb1507e6/recovered.edits/4.seqid to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/archive/data/ns3/table3_restore/504ca2b90001e75c082d0a09cb1507e6/recovered.edits/4.seqid 2023-10-02T12:32:08,000 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/table3_restore/504ca2b90001e75c082d0a09cb1507e6 2023-10-02T12:32:08,000 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(293): Archived ns3:table3_restore regions 2023-10-02T12:32:08,002 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=152, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-10-02T12:32:08,004 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:table3_restore from hbase:meta 2023-10-02T12:32:08,005 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:table3_restore' descriptor. 2023-10-02T12:32:08,006 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=152, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-10-02T12:32:08,006 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:table3_restore' from region states. 2023-10-02T12:32:08,007 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696249928006"}]},"ts":"9223372036854775807"} 2023-10-02T12:32:08,008 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:32:08,008 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 504ca2b90001e75c082d0a09cb1507e6, NAME => 'ns3:table3_restore,,1696249926145.504ca2b90001e75c082d0a09cb1507e6.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:32:08,008 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:table3_restore' as deleted. 2023-10-02T12:32:08,008 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1696249928008"}]},"ts":"9223372036854775807"} 2023-10-02T12:32:08,010 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:table3_restore state from META 2023-10-02T12:32:08,012 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(127): Finished pid=152, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-10-02T12:32:08,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=152, state=SUCCESS; DeleteTableProcedure table=ns3:table3_restore in 25 msec 2023-10-02T12:32:08,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=152 2023-10-02T12:32:08,112 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleCommand Thread=603 (was 606), OpenFileDescriptor=682 (was 683), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=476 (was 539), ProcessCount=181 (was 181), AvailableMemoryMB=2337 (was 2896) 2023-10-02T12:32:08,112 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=603 is superior to 500 2023-10-02T12:32:08,133 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleDNE Thread=603, OpenFileDescriptor=682, MaxFileDescriptor=60000, SystemLoadAverage=476, ProcessCount=181, AvailableMemoryMB=2331 2023-10-02T12:32:08,133 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=603 is superior to 500 2023-10-02T12:32:08,133 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestFullRestore(260): test restore fails on a single table that does not exist 2023-10-02T12:32:08,139 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70a3bcf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:08,141 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:08,143 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49058, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:08,144 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:32:08,145 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:51386, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:32:08,162 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:32:08,167 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:32:08,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249928167 (current time:1696249928167). 2023-10-02T12:32:08,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:32:08,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:32:08,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ccf38cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:08,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:08,173 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:49064, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:08,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:08,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:32:08,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=153, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:32:08,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 153 2023-10-02T12:32:08,177 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:32:08,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=153 2023-10-02T12:32:08,178 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:32:08,181 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor129.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:32:08,183 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:32:08,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=153 2023-10-02T12:32:08,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=153 2023-10-02T12:32:08,590 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:32:08,590 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987}] 2023-10-02T12:32:08,592 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=154, ppid=153, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:32:08,608 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:32:08,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=154 2023-10-02T12:32:08,743 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:32:08,744 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegion(2819): Flushing 7ec576d712d0f3b0c0592584b88ce987 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-10-02T12:32:08,748 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/d0f3dcf500344358956a19104aad9e8f is 171, key is trslm:hdfs://localhost:37823/backupUT\x00ns2:test-16962497916551/meta:log-roll-map/1696249922145/Put/seqid=0 2023-10-02T12:32:08,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=153 2023-10-02T12:32:09,152 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/d0f3dcf500344358956a19104aad9e8f 2023-10-02T12:32:09,165 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/eac7dd6dd5764b7f91b006a5dc2e0b23 is 524, key is session:backup_1696249912589/session:context/1696249922597/Put/seqid=0 2023-10-02T12:32:09,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=153 2023-10-02T12:32:09,570 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/eac7dd6dd5764b7f91b006a5dc2e0b23 2023-10-02T12:32:09,577 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/d0f3dcf500344358956a19104aad9e8f as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/d0f3dcf500344358956a19104aad9e8f 2023-10-02T12:32:09,582 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/d0f3dcf500344358956a19104aad9e8f, entries=6, sequenceid=116, filesize=5.8 K 2023-10-02T12:32:09,583 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/eac7dd6dd5764b7f91b006a5dc2e0b23 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/eac7dd6dd5764b7f91b006a5dc2e0b23 2023-10-02T12:32:09,588 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/eac7dd6dd5764b7f91b006a5dc2e0b23, entries=2, sequenceid=116, filesize=5.5 K 2023-10-02T12:32:09,589 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for 7ec576d712d0f3b0c0592584b88ce987 in 846ms, sequenceid=116, compaction requested=true 2023-10-02T12:32:09,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegion(2520): Flush status journal for 7ec576d712d0f3b0c0592584b88ce987: 2023-10-02T12:32:09,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. for snapshot_backup_system completed. 2023-10-02T12:32:09,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:32:09,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:32:09,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/85d2568c07cc4be686b8e706b89254ce, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/9bf0c2ed3e5c43ba8bb2287a7e2731fa, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b4a8e50177b941698e5089bffe91b251, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/23e51484216f4e7a9dca73d1bb9d8f09, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/f1ceb39b31074c03be29750a860f2cc8, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/d0f3dcf500344358956a19104aad9e8f] hfiles 2023-10-02T12:32:09,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (1/10): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98 for snapshot=snapshot_backup_system 2023-10-02T12:32:09,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (2/10): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255 for snapshot=snapshot_backup_system 2023-10-02T12:32:09,589 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (3/10): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8 for snapshot=snapshot_backup_system 2023-10-02T12:32:09,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (4/10): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/85d2568c07cc4be686b8e706b89254ce for snapshot=snapshot_backup_system 2023-10-02T12:32:09,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (5/10): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5 for snapshot=snapshot_backup_system 2023-10-02T12:32:09,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (6/10): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/9bf0c2ed3e5c43ba8bb2287a7e2731fa for snapshot=snapshot_backup_system 2023-10-02T12:32:09,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (7/10): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b4a8e50177b941698e5089bffe91b251 for snapshot=snapshot_backup_system 2023-10-02T12:32:09,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (8/10): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/23e51484216f4e7a9dca73d1bb9d8f09 for snapshot=snapshot_backup_system 2023-10-02T12:32:09,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (9/10): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/f1ceb39b31074c03be29750a860f2cc8 for snapshot=snapshot_backup_system 2023-10-02T12:32:09,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (10/10): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/d0f3dcf500344358956a19104aad9e8f for snapshot=snapshot_backup_system 2023-10-02T12:32:09,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/f8db732eb5cd417eaa236f25fd437245, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/fe4c59a0e2584ff5980ec23dd90e1c0b, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/2129c391bd7447f0b05fea1084f2d47d, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/120da2f382584d43adc4417dfe387fae, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/75d622dd3189402a9723a0642f3437f9, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/56ef49f7bed44dd1b83792cad64956f0, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/eac7dd6dd5764b7f91b006a5dc2e0b23] hfiles 2023-10-02T12:32:09,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (1/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0 for snapshot=snapshot_backup_system 2023-10-02T12:32:09,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (2/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c for snapshot=snapshot_backup_system 2023-10-02T12:32:09,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (3/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229 for snapshot=snapshot_backup_system 2023-10-02T12:32:09,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (4/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423 for snapshot=snapshot_backup_system 2023-10-02T12:32:09,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (5/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/f8db732eb5cd417eaa236f25fd437245 for snapshot=snapshot_backup_system 2023-10-02T12:32:09,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (6/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/fe4c59a0e2584ff5980ec23dd90e1c0b for snapshot=snapshot_backup_system 2023-10-02T12:32:09,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (7/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/2129c391bd7447f0b05fea1084f2d47d for snapshot=snapshot_backup_system 2023-10-02T12:32:09,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (8/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/120da2f382584d43adc4417dfe387fae for snapshot=snapshot_backup_system 2023-10-02T12:32:09,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (9/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/75d622dd3189402a9723a0642f3437f9 for snapshot=snapshot_backup_system 2023-10-02T12:32:09,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (10/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/56ef49f7bed44dd1b83792cad64956f0 for snapshot=snapshot_backup_system 2023-10-02T12:32:09,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (11/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/eac7dd6dd5764b7f91b006a5dc2e0b23 for snapshot=snapshot_backup_system 2023-10-02T12:32:09,998 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:32:09,999 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=154 2023-10-02T12:32:09,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=154 2023-10-02T12:32:09,999 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:32:10,001 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=154, ppid=153, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:32:10,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=154, resume processing ppid=153 2023-10-02T12:32:10,004 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:32:10,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=154, ppid=153, state=SUCCESS; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 in 1.4120 sec 2023-10-02T12:32:10,004 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:32:10,005 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:32:10,005 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:32:10,006 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:32:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=153 2023-10-02T12:32:10,424 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:32:10,449 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:32:10,449 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:32:10,453 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:32:10,454 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 153 2023-10-02T12:32:10,455 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=153, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2800 sec 2023-10-02T12:32:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=153 2023-10-02T12:32:12,290 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(115): Backup backup_1696249928139 started at 1696249932289. 2023-10-02T12:32:12,291 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(119): Backup session backup_1696249928139 has been started. 2023-10-02T12:32:12,300 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:32:12,302 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:32:12,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:32:12,302 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:32:12,302 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:32:12,304 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:32:12,304 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:32:12,304 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:32:12,306 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:32:12,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:32:12,306 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:12,306 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:32:12,306 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:32:12,307 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:32:12,307 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:32:12,308 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:32:12,308 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:32:12,308 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:32:12,308 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:32:12,308 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:32:12,309 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:32:12,309 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:32:12,309 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:32:12,313 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:12,313 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:32:12,313 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:32:12,313 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:32:12,313 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:32:12,313 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:32:12,314 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:32:12,314 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:32:12,316 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:12,316 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:32:12,317 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:32:12,317 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@23fa624c[Count = 0] remaining members to acquire global barrier 2023-10-02T12:32:12,317 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:32:12,317 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:32:12,318 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:32:12,322 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:32:12,322 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:32:12,322 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:32:12,322 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:32:12,322 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:12,322 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696249914878 highest: 1696249914878 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:12,322 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114:(num 1696249914878) roll requested 2023-10-02T12:32:12,340 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249932323, exclude list is [], retry=0 2023-10-02T12:32:12,344 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:32:12,347 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249914878 with entries=15, filesize=4.05 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249932323 2023-10-02T12:32:12,347 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:32:12,347 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249914878 is not closed yet, will try archiving it next time 2023-10-02T12:32:12,347 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta:.meta(num 1696249914907) roll requested 2023-10-02T12:32:12,353 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249914878 not finished, retry = 0 2023-10-02T12:32:12,368 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249932347.meta, exclude list is [], retry=0 2023-10-02T12:32:12,372 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:32:12,380 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249914907.meta with entries=24, filesize=5.77 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249932347.meta 2023-10-02T12:32:12,380 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:32:12,380 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249914907.meta is not closed yet, will try archiving it next time 2023-10-02T12:32:12,383 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-10-02T12:32:12,383 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696249932323 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:12,392 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:32:12,392 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,38255,1696249782114' in zk 2023-10-02T12:32:12,396 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:32:12,396 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:32:12,396 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:12,396 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:32:12,397 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:32:12,397 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:32:12,397 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:32:12,398 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:32:12,398 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:32:12,398 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:12,399 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:32:12,399 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:32:12,399 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:12,400 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,38255,1696249782114': 2023-10-02T12:32:12,400 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:32:12,400 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:32:12,400 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:32:12,400 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:32:12,400 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:32:12,400 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:32:12,402 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:32:12,402 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:32:12,402 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:32:12,403 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:32:12,403 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:32:12,403 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:32:12,403 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:32:12,403 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:32:12,404 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:32:12,404 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:32:12,405 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:32:12,406 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:32:12,407 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:12,407 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:32:12,407 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:32:12,408 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:12,412 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:32:12,413 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:32:12,413 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:32:12,416 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:32:12,416 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:32:12,417 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:32:12,417 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:32:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:32:12,519 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249932518_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } 2023-10-02T12:32:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249932519 (current time:1696249932519). 2023-10-02T12:32:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:32:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249932518_default_test-1696249791655 VERSION not specified, setting to 2 2023-10-02T12:32:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b3296a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:12,544 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33224, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:32:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=155, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1696249932518_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } 2023-10-02T12:32:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249932518_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 }, snapshot procedure id = 155 2023-10-02T12:32:12,550 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1696249932518_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:32:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=155 2023-10-02T12:32:12,551 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1696249932518_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:32:12,555 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249932518_default_test-1696249791655 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor129.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:32:12,559 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1696249932518_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:32:12,572 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1696249932518_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:32:12,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE; SnapshotRegionProcedure 35dcfa6f2558560536edbd70539305b1}] 2023-10-02T12:32:12,578 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=156, ppid=155, state=RUNNABLE; SnapshotRegionProcedure 35dcfa6f2558560536edbd70539305b1 2023-10-02T12:32:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=155 2023-10-02T12:32:12,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=156 2023-10-02T12:32:12,730 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1. 2023-10-02T12:32:12,730 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.HRegion(2520): Flush status journal for 35dcfa6f2558560536edbd70539305b1: 2023-10-02T12:32:12,731 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1. for snapshot_1696249932518_default_test-1696249791655 completed. 2023-10-02T12:32:12,731 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(240): Storing 'test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1.' region-info for snapshot=snapshot_1696249932518_default_test-1696249791655 2023-10-02T12:32:12,731 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:32:12,731 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1/f/553dd16623454c9282013d29ef554af3_SeqId_4_] hfiles 2023-10-02T12:32:12,731 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1/f/553dd16623454c9282013d29ef554af3_SeqId_4_ for snapshot=snapshot_1696249932518_default_test-1696249791655 2023-10-02T12:32:12,740 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1. 2023-10-02T12:32:12,740 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=156 2023-10-02T12:32:12,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=156 2023-10-02T12:32:12,740 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249932518_default_test-1696249791655 on region 35dcfa6f2558560536edbd70539305b1 2023-10-02T12:32:12,742 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=156, ppid=155, state=RUNNABLE; SnapshotRegionProcedure 35dcfa6f2558560536edbd70539305b1 2023-10-02T12:32:12,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=156, resume processing ppid=155 2023-10-02T12:32:12,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=156, ppid=155, state=SUCCESS; SnapshotRegionProcedure 35dcfa6f2558560536edbd70539305b1 in 170 msec 2023-10-02T12:32:12,746 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1696249932518_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:32:12,747 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1696249932518_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:32:12,748 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1696249932518_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:32:12,748 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249932518_default_test-1696249791655 2023-10-02T12:32:12,749 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249932518_default_test-1696249791655 2023-10-02T12:32:12,772 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1696249932518_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:32:12,778 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1696249932518_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:32:12,779 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249932518_default_test-1696249791655 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249932518_default_test-1696249791655 2023-10-02T12:32:12,781 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1696249932518_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:32:12,781 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249932518_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 }, snapshot procedure id = 155 2023-10-02T12:32:12,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=155, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1696249932518_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } in 235 msec 2023-10-02T12:32:12,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=155 2023-10-02T12:32:12,855 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696249928139 2023-10-02T12:32:12,855 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:32:12,855 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-10-02T12:32:12,855 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249928139_test-1696249791655 2023-10-02T12:32:12,855 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249932518_default_test-1696249791655 to hdfs://localhost:37823/backupUT/backup_1696249928139/default/test-1696249791655/ 2023-10-02T12:32:12,855 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:32:12,891 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:32:12,891 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,40619,1696249778965_-91867442_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37823/backupUT/backup_1696249928139/default/test-1696249791655, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37823/backupUT/backup_1696249928139/default/test-1696249791655/.hbase-snapshot/.tmp/snapshot_1696249932518_default_test-1696249791655 2023-10-02T12:32:12,891 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07, snapshotDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249932518_default_test-1696249791655. 2023-10-02T12:32:12,901 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249932518_default_test-1696249791655 to hdfs://localhost:37823/backupUT/backup_1696249928139/default/test-1696249791655/.hbase-snapshot/.tmp/snapshot_1696249932518_default_test-1696249791655 2023-10-02T12:32:13,349 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:13,349 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:13,350 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:13,350 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:14,583 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-1533817044797560.jar 2023-10-02T12:32:14,584 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:14,584 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:14,585 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:14,586 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:14,586 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:14,587 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:14,587 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:14,588 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:14,588 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:14,589 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:32:14,590 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:32:14,591 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:32:14,591 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:32:14,592 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:32:14,593 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:32:14,593 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:32:14,594 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:32:14,595 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:32:14,595 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:32:14,596 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:32:14,596 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:32:14,597 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:32:14,598 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:32:14,599 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:32:14,600 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:32:14,600 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:32:14,601 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:32:14,602 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:32:14,662 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249932518_default_test-1696249791655' hfile list 2023-10-02T12:32:14,664 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-10-02T12:32:14,755 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249934707/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:32:14,756 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249934707/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:32:14,835 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:32:14,862 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1/f/553dd16623454c9282013d29ef554af3_SeqId_4_ output=hdfs://localhost:37823/backupUT/backup_1696249928139/default/test-1696249791655/archive/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1/f/553dd16623454c9282013d29ef554af3_SeqId_4_ 2023-10-02T12:32:14,862 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-10-02T12:32:15,804 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:32:15,805 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:32:15,813 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249932518_default_test-1696249791655 2023-10-02T12:32:15,813 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249932518_default_test-1696249791655 finished. 2023-10-02T12:32:15,813 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1034): test-1696249791655 2023-10-02T12:32:15,822 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249928139 2023-10-02T12:32:15,822 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:32:15,828 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37823/backupUT/backup_1696249928139/.backup.manifest 2023-10-02T12:32:15,828 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(358): Backup backup_1696249928139 finished: type=FULL,tablelist=test-1696249791655,targetRootDir=hdfs://localhost:37823/backupUT,startts=1696249932289,completets=1696249935822,bytescopied=0 2023-10-02T12:32:15,828 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:32:15,828 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249932518_default_test-1696249791655 2023-10-02T12:32:15,829 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249932518_default_test-1696249791655" 2023-10-02T12:32:15,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249932518_default_test-1696249791655 2023-10-02T12:32:15,833 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249932518_default_test-1696249791655 for backup backup_1696249928139 succeeded. 2023-10-02T12:32:15,833 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:32:15,839 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:32:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:32:15,842 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:32:15,844 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:32:15,849 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(376): Backup backup_1696249928139 completed. 2023-10-02T12:32:15,850 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:15,871 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestFullRestore(265): backup complete 2023-10-02T12:32:15,872 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-10-02T12:32:15,872 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249928139/.backup.manifest 2023-10-02T12:32:15,875 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249928139/.backup.manifest 2023-10-02T12:32:15,876 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249928139/.backup.manifest 2023-10-02T12:32:15,878 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249928139/.backup.manifest 2023-10-02T12:32:15,878 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(148): Restoring 'faketable' to 'table1' from full backup image hdfs://localhost:37823/backupUT/backup_1696249928139/default/faketable 2023-10-02T12:32:15,905 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleDNE Thread=604 (was 603) - Thread LEAK? -, OpenFileDescriptor=697 (was 682) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=503 (was 476) - SystemLoadAverage LEAK? -, ProcessCount=181 (was 181), AvailableMemoryMB=2109 (was 2331) 2023-10-02T12:32:15,905 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-10-02T12:32:15,932 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleCommand Thread=604, OpenFileDescriptor=697, MaxFileDescriptor=60000, SystemLoadAverage=503, ProcessCount=181, AvailableMemoryMB=2108 2023-10-02T12:32:15,932 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-10-02T12:32:15,932 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestFullRestore(76): test full restore on a single table empty table: command-line 2023-10-02T12:32:15,939 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fe92dc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:15,942 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:15,943 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33238, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:15,949 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:32:15,950 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:55914, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:32:15,986 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:32:16,001 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:32:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249936001 (current time:1696249936001). 2023-10-02T12:32:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:32:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:32:16,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f0a3111, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:16,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:16,014 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33240, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:16,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:32:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=157, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:32:16,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 157 2023-10-02T12:32:16,024 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:32:16,028 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:32:16,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=157 2023-10-02T12:32:16,032 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor129.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:32:16,036 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:32:16,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=157 2023-10-02T12:32:16,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=157 2023-10-02T12:32:16,446 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:32:16,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987}] 2023-10-02T12:32:16,449 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=158, ppid=157, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:32:16,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=158 2023-10-02T12:32:16,601 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:32:16,602 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegion(2819): Flushing 7ec576d712d0f3b0c0592584b88ce987 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-10-02T12:32:16,605 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/af19b5a70b5a4fe19a14d9bf11f1ca14 is 169, key is trslm:hdfs://localhost:37823/backupUT\x00test-1696249791655/meta:log-roll-map/1696249935816/Put/seqid=0 2023-10-02T12:32:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=157 2023-10-02T12:32:17,011 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/af19b5a70b5a4fe19a14d9bf11f1ca14 2023-10-02T12:32:17,021 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/6eccbee667ca41b9851e81d99bcb3699 is 452, key is session:backup_1696249928139/session:context/1696249935843/Put/seqid=0 2023-10-02T12:32:17,033 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/6eccbee667ca41b9851e81d99bcb3699 2023-10-02T12:32:17,051 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/af19b5a70b5a4fe19a14d9bf11f1ca14 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/af19b5a70b5a4fe19a14d9bf11f1ca14 2023-10-02T12:32:17,064 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/af19b5a70b5a4fe19a14d9bf11f1ca14, entries=4, sequenceid=127, filesize=5.5 K 2023-10-02T12:32:17,066 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/6eccbee667ca41b9851e81d99bcb3699 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/6eccbee667ca41b9851e81d99bcb3699 2023-10-02T12:32:17,073 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/6eccbee667ca41b9851e81d99bcb3699, entries=2, sequenceid=127, filesize=5.4 K 2023-10-02T12:32:17,078 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 7ec576d712d0f3b0c0592584b88ce987 in 476ms, sequenceid=127, compaction requested=true 2023-10-02T12:32:17,078 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegion(2520): Flush status journal for 7ec576d712d0f3b0c0592584b88ce987: 2023-10-02T12:32:17,078 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. for snapshot_backup_system completed. 2023-10-02T12:32:17,078 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:32:17,078 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:32:17,078 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/85d2568c07cc4be686b8e706b89254ce, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/9bf0c2ed3e5c43ba8bb2287a7e2731fa, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b4a8e50177b941698e5089bffe91b251, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/23e51484216f4e7a9dca73d1bb9d8f09, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/f1ceb39b31074c03be29750a860f2cc8, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/d0f3dcf500344358956a19104aad9e8f, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/af19b5a70b5a4fe19a14d9bf11f1ca14] hfiles 2023-10-02T12:32:17,078 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (1/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98 for snapshot=snapshot_backup_system 2023-10-02T12:32:17,078 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (2/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255 for snapshot=snapshot_backup_system 2023-10-02T12:32:17,078 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (3/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8 for snapshot=snapshot_backup_system 2023-10-02T12:32:17,078 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (4/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/85d2568c07cc4be686b8e706b89254ce for snapshot=snapshot_backup_system 2023-10-02T12:32:17,078 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (5/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5 for snapshot=snapshot_backup_system 2023-10-02T12:32:17,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (6/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/9bf0c2ed3e5c43ba8bb2287a7e2731fa for snapshot=snapshot_backup_system 2023-10-02T12:32:17,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (7/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b4a8e50177b941698e5089bffe91b251 for snapshot=snapshot_backup_system 2023-10-02T12:32:17,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (8/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/23e51484216f4e7a9dca73d1bb9d8f09 for snapshot=snapshot_backup_system 2023-10-02T12:32:17,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (9/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/f1ceb39b31074c03be29750a860f2cc8 for snapshot=snapshot_backup_system 2023-10-02T12:32:17,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (10/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/d0f3dcf500344358956a19104aad9e8f for snapshot=snapshot_backup_system 2023-10-02T12:32:17,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (11/11): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/af19b5a70b5a4fe19a14d9bf11f1ca14 for snapshot=snapshot_backup_system 2023-10-02T12:32:17,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/f8db732eb5cd417eaa236f25fd437245, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/fe4c59a0e2584ff5980ec23dd90e1c0b, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/2129c391bd7447f0b05fea1084f2d47d, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/120da2f382584d43adc4417dfe387fae, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/75d622dd3189402a9723a0642f3437f9, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/56ef49f7bed44dd1b83792cad64956f0, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/eac7dd6dd5764b7f91b006a5dc2e0b23, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/6eccbee667ca41b9851e81d99bcb3699] hfiles 2023-10-02T12:32:17,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (1/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0 for snapshot=snapshot_backup_system 2023-10-02T12:32:17,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (2/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c for snapshot=snapshot_backup_system 2023-10-02T12:32:17,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (3/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229 for snapshot=snapshot_backup_system 2023-10-02T12:32:17,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (4/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423 for snapshot=snapshot_backup_system 2023-10-02T12:32:17,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (5/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/f8db732eb5cd417eaa236f25fd437245 for snapshot=snapshot_backup_system 2023-10-02T12:32:17,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (6/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/fe4c59a0e2584ff5980ec23dd90e1c0b for snapshot=snapshot_backup_system 2023-10-02T12:32:17,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (7/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/2129c391bd7447f0b05fea1084f2d47d for snapshot=snapshot_backup_system 2023-10-02T12:32:17,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (8/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/120da2f382584d43adc4417dfe387fae for snapshot=snapshot_backup_system 2023-10-02T12:32:17,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (9/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/75d622dd3189402a9723a0642f3437f9 for snapshot=snapshot_backup_system 2023-10-02T12:32:17,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (10/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/56ef49f7bed44dd1b83792cad64956f0 for snapshot=snapshot_backup_system 2023-10-02T12:32:17,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (11/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/eac7dd6dd5764b7f91b006a5dc2e0b23 for snapshot=snapshot_backup_system 2023-10-02T12:32:17,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (12/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/6eccbee667ca41b9851e81d99bcb3699 for snapshot=snapshot_backup_system 2023-10-02T12:32:17,089 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:32:17,089 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=158 2023-10-02T12:32:17,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=158 2023-10-02T12:32:17,090 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:32:17,091 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=158, ppid=157, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:32:17,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=158, resume processing ppid=157 2023-10-02T12:32:17,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=158, ppid=157, state=SUCCESS; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 in 646 msec 2023-10-02T12:32:17,094 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:32:17,094 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:32:17,095 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:32:17,095 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:32:17,096 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:32:17,112 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:32:17,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=157 2023-10-02T12:32:17,206 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:32:17,207 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:32:17,209 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:32:17,209 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 157 2023-10-02T12:32:17,210 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=157, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.1920 sec 2023-10-02T12:32:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=157 2023-10-02T12:32:18,148 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(115): Backup backup_1696249935939 started at 1696249938148. 2023-10-02T12:32:18,150 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(119): Backup session backup_1696249935939 has been started. 2023-10-02T12:32:18,155 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:32:18,156 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:32:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-10-02T12:32:18,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:32:18,156 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:32:18,156 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:32:18,157 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:32:18,157 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:32:18,158 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:32:18,159 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:18,159 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:32:18,159 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:32:18,160 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:32:18,160 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:32:18,160 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:32:18,160 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:32:18,160 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:32:18,164 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:32:18,164 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-10-02T12:32:18,164 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:32:18,164 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:32:18,164 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:32:18,165 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:32:18,165 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:32:18,165 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:32:18,168 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:32:18,168 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:18,168 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:32:18,168 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:32:18,168 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:32:18,169 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:32:18,169 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:32:18,169 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:32:18,170 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:18,170 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:32:18,171 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:32:18,171 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@33e72673[Count = 0] remaining members to acquire global barrier 2023-10-02T12:32:18,171 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:32:18,171 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:32:18,172 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:32:18,172 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:32:18,172 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:32:18,172 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:32:18,172 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:32:18,173 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:18,173 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696249932323 highest: 1696249932323 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:18,173 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114:(num 1696249932323) roll requested 2023-10-02T12:32:18,185 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249938173, exclude list is [], retry=0 2023-10-02T12:32:18,188 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:32:18,190 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249932323 with entries=10, filesize=2.31 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249938173 2023-10-02T12:32:18,190 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:32:18,190 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249932323 is not closed yet, will try archiving it next time 2023-10-02T12:32:18,190 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta:.meta(num 1696249932347) roll requested 2023-10-02T12:32:18,190 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249914878 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/oldWALs/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249914878 2023-10-02T12:32:18,193 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249932323 not finished, retry = 0 2023-10-02T12:32:18,205 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249938190.meta, exclude list is [], retry=0 2023-10-02T12:32:18,208 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:32:18,210 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249932347.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249938190.meta 2023-10-02T12:32:18,210 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:32:18,210 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249932347.meta is not closed yet, will try archiving it next time 2023-10-02T12:32:18,213 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-10-02T12:32:18,213 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696249938173 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:18,215 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249932347.meta to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/oldWALs/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249932347.meta 2023-10-02T12:32:18,227 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:32:18,227 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,38255,1696249782114' in zk 2023-10-02T12:32:18,231 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:32:18,231 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:18,231 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:32:18,231 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:32:18,231 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:32:18,231 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:32:18,232 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:32:18,233 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:32:18,233 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:32:18,233 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:18,234 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:32:18,234 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:32:18,235 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:18,235 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,38255,1696249782114': 2023-10-02T12:32:18,235 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:32:18,235 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:32:18,236 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:32:18,236 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:32:18,236 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:32:18,236 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:32:18,238 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:32:18,238 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:32:18,238 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:32:18,238 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:32:18,238 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:32:18,238 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:32:18,238 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:32:18,238 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:32:18,239 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:32:18,239 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:32:18,239 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:32:18,239 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:32:18,240 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:18,240 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:32:18,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:32:18,241 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:18,249 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:32:18,249 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:32:18,249 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:32:18,249 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:32:18,249 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:32:18,250 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:32:18,250 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:32:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:32:18,355 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249938354_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } 2023-10-02T12:32:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249938355 (current time:1696249938355). 2023-10-02T12:32:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:32:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249938354_default_test-1696249791655 VERSION not specified, setting to 2 2023-10-02T12:32:18,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8637d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:18,375 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:33244, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:32:18,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=159, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1696249938354_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } 2023-10-02T12:32:18,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249938354_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 }, snapshot procedure id = 159 2023-10-02T12:32:18,380 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1696249938354_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:32:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=159 2023-10-02T12:32:18,381 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1696249938354_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:32:18,384 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249938354_default_test-1696249791655 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor129.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:32:18,387 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1696249938354_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:32:18,419 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1696249938354_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:32:18,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE; SnapshotRegionProcedure 35dcfa6f2558560536edbd70539305b1}] 2023-10-02T12:32:18,422 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=160, ppid=159, state=RUNNABLE; SnapshotRegionProcedure 35dcfa6f2558560536edbd70539305b1 2023-10-02T12:32:18,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=159 2023-10-02T12:32:18,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=160 2023-10-02T12:32:18,573 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1. 2023-10-02T12:32:18,574 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.HRegion(2520): Flush status journal for 35dcfa6f2558560536edbd70539305b1: 2023-10-02T12:32:18,574 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1. for snapshot_1696249938354_default_test-1696249791655 completed. 2023-10-02T12:32:18,574 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(240): Storing 'test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1.' region-info for snapshot=snapshot_1696249938354_default_test-1696249791655 2023-10-02T12:32:18,574 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:32:18,574 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1/f/553dd16623454c9282013d29ef554af3_SeqId_4_] hfiles 2023-10-02T12:32:18,574 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1/f/553dd16623454c9282013d29ef554af3_SeqId_4_ for snapshot=snapshot_1696249938354_default_test-1696249791655 2023-10-02T12:32:18,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=159 2023-10-02T12:32:18,984 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1. 2023-10-02T12:32:18,984 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=160 2023-10-02T12:32:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=160 2023-10-02T12:32:18,984 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249938354_default_test-1696249791655 on region 35dcfa6f2558560536edbd70539305b1 2023-10-02T12:32:18,986 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=160, ppid=159, state=RUNNABLE; SnapshotRegionProcedure 35dcfa6f2558560536edbd70539305b1 2023-10-02T12:32:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=159 2023-10-02T12:32:18,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=160, resume processing ppid=159 2023-10-02T12:32:18,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=160, ppid=159, state=SUCCESS; SnapshotRegionProcedure 35dcfa6f2558560536edbd70539305b1 in 575 msec 2023-10-02T12:32:18,996 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1696249938354_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:32:18,997 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1696249938354_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:32:18,998 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1696249938354_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:32:18,998 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249938354_default_test-1696249791655 2023-10-02T12:32:18,999 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249938354_default_test-1696249791655 2023-10-02T12:32:19,024 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1696249938354_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:32:19,028 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1696249938354_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:32:19,029 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249938354_default_test-1696249791655 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249938354_default_test-1696249791655 2023-10-02T12:32:19,030 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1696249938354_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:32:19,031 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249938354_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 }, snapshot procedure id = 159 2023-10-02T12:32:19,032 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=159, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1696249938354_default_test-1696249791655 table=test-1696249791655 type=FLUSH ttl=0 } in 655 msec 2023-10-02T12:32:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=159 2023-10-02T12:32:19,488 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696249935939 2023-10-02T12:32:19,488 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:32:19,488 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-10-02T12:32:19,488 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249935939_test-1696249791655 2023-10-02T12:32:19,488 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249938354_default_test-1696249791655 to hdfs://localhost:37823/backupUT/backup_1696249935939/default/test-1696249791655/ 2023-10-02T12:32:19,488 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:32:19,519 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:32:19,519 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,40619,1696249778965_-196973771_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37823/backupUT/backup_1696249935939/default/test-1696249791655, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37823/backupUT/backup_1696249935939/default/test-1696249791655/.hbase-snapshot/.tmp/snapshot_1696249938354_default_test-1696249791655 2023-10-02T12:32:19,519 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07, snapshotDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249938354_default_test-1696249791655. 2023-10-02T12:32:19,527 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249938354_default_test-1696249791655 to hdfs://localhost:37823/backupUT/backup_1696249935939/default/test-1696249791655/.hbase-snapshot/.tmp/snapshot_1696249938354_default_test-1696249791655 2023-10-02T12:32:19,986 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:19,986 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:19,987 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:19,988 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:21,034 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-8195712577895849261.jar 2023-10-02T12:32:21,035 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:21,035 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:21,035 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:21,036 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:21,036 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:21,036 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:21,037 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:21,037 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:21,037 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:21,038 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:32:21,038 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:32:21,039 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:32:21,039 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:32:21,040 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:32:21,040 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:32:21,040 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:32:21,040 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:32:21,041 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:32:21,041 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:32:21,042 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:32:21,042 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:32:21,043 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:32:21,043 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:32:21,043 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:32:21,044 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:32:21,044 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:32:21,044 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:32:21,045 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:32:21,088 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249938354_default_test-1696249791655' hfile list 2023-10-02T12:32:21,090 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-10-02T12:32:21,161 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249941125/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:32:21,162 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249941125/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:32:21,217 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:32:21,241 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1/f/553dd16623454c9282013d29ef554af3_SeqId_4_ output=hdfs://localhost:37823/backupUT/backup_1696249935939/default/test-1696249791655/archive/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1/f/553dd16623454c9282013d29ef554af3_SeqId_4_ 2023-10-02T12:32:21,241 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-10-02T12:32:22,192 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:32:22,193 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:32:22,205 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249938354_default_test-1696249791655 2023-10-02T12:32:22,205 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249938354_default_test-1696249791655 finished. 2023-10-02T12:32:22,205 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1034): test-1696249791655 2023-10-02T12:32:22,211 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249935939 2023-10-02T12:32:22,211 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:32:22,617 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37823/backupUT/backup_1696249935939/.backup.manifest 2023-10-02T12:32:22,617 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(358): Backup backup_1696249935939 finished: type=FULL,tablelist=test-1696249791655,targetRootDir=hdfs://localhost:37823/backupUT,startts=1696249938148,completets=1696249942211,bytescopied=0 2023-10-02T12:32:22,617 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:32:22,618 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249938354_default_test-1696249791655 2023-10-02T12:32:22,618 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249938354_default_test-1696249791655" 2023-10-02T12:32:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249938354_default_test-1696249791655 2023-10-02T12:32:22,622 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249938354_default_test-1696249791655 for backup backup_1696249935939 succeeded. 2023-10-02T12:32:22,622 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:32:22,627 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:32:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:32:22,629 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:32:22,631 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:32:22,637 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(376): Backup backup_1696249935939 completed. 2023-10-02T12:32:22,637 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:22,637 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestFullRestore(80): backup complete 2023-10-02T12:32:22,656 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@570babf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:22,658 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:22,659 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57500, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:22,660 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-10-02T12:32:22,660 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249935939/.backup.manifest 2023-10-02T12:32:22,662 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249935939/.backup.manifest 2023-10-02T12:32:22,663 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249935939/.backup.manifest 2023-10-02T12:32:22,665 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249935939/.backup.manifest 2023-10-02T12:32:22,665 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(148): Restoring 'test-1696249791655' to 'table1' from full backup image hdfs://localhost:37823/backupUT/backup_1696249935939/default/test-1696249791655 2023-10-02T12:32:22,672 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(487): Creating target table 'table1' 2023-10-02T12:32:22,672 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:37823/backupUT/backup_1696249935939/default/test-1696249791655/archive/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1 2023-10-02T12:32:22,673 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:37823/backupUT/backup_1696249935939/default/test-1696249791655/archive/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1/f in region [hdfs://localhost:37823/backupUT/backup_1696249935939/default/test-1696249791655/archive/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1] 2023-10-02T12:32:22,676 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:37823/backupUT/backup_1696249935939/default/test-1696249791655/archive/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1/f/553dd16623454c9282013d29ef554af3_SeqId_4_ first=row0 last=row98 2023-10-02T12:32:22,677 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:32:22,678 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48556, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:32:22,679 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$4(2332): Client=jenkins//172.31.12.81 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-10-02T12:32:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=161, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-10-02T12:32:22,683 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-10-02T12:32:22,684 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(710): Client=jenkins//172.31.12.81 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 161 2023-10-02T12:32:22,685 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-10-02T12:32:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=161 2023-10-02T12:32:22,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=161 2023-10-02T12:32:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=161 2023-10-02T12:32:23,101 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 29c0434c6d75bd4ced8d82f432a6f81d, NAME => 'table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:32:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=161 2023-10-02T12:32:23,508 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:32:23,508 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing 29c0434c6d75bd4ced8d82f432a6f81d, disabling compactions & flushes 2023-10-02T12:32:23,508 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d. 2023-10-02T12:32:23,508 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d. 2023-10-02T12:32:23,508 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d. after waiting 0 ms 2023-10-02T12:32:23,508 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d. 2023-10-02T12:32:23,508 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d. 2023-10-02T12:32:23,508 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for 29c0434c6d75bd4ced8d82f432a6f81d: 2023-10-02T12:32:23,509 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-10-02T12:32:23,509 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1696249943509"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1696249943509"}]},"ts":"1696249943509"} 2023-10-02T12:32:23,510 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-10-02T12:32:23,511 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-10-02T12:32:23,511 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249943511"}]},"ts":"1696249943511"} 2023-10-02T12:32:23,512 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-10-02T12:32:23,517 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=29c0434c6d75bd4ced8d82f432a6f81d, ASSIGN}] 2023-10-02T12:32:23,519 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=29c0434c6d75bd4ced8d82f432a6f81d, ASSIGN 2023-10-02T12:32:23,519 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=29c0434c6d75bd4ced8d82f432a6f81d, ASSIGN; state=OFFLINE, location=jenkins-hbase3.apache.org,38255,1696249782114; forceNewPlan=false, retain=false 2023-10-02T12:32:23,673 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=162 updating hbase:meta row=29c0434c6d75bd4ced8d82f432a6f81d, regionState=OPENING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:23,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE; OpenRegionProcedure 29c0434c6d75bd4ced8d82f432a6f81d, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:32:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=161 2023-10-02T12:32:23,832 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] handler.AssignRegionHandler(130): Open table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d. 2023-10-02T12:32:23,832 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7254): Opening region: {ENCODED => 29c0434c6d75bd4ced8d82f432a6f81d, NAME => 'table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d.', STARTKEY => '', ENDKEY => ''} 2023-10-02T12:32:23,832 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-10-02T12:32:23,832 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 29c0434c6d75bd4ced8d82f432a6f81d 2023-10-02T12:32:23,832 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(888): Instantiated table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-10-02T12:32:23,833 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7296): checking encryption for 29c0434c6d75bd4ced8d82f432a6f81d 2023-10-02T12:32:23,833 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7299): checking classloading for 29c0434c6d75bd4ced8d82f432a6f81d 2023-10-02T12:32:23,834 INFO [StoreOpener-29c0434c6d75bd4ced8d82f432a6f81d-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 29c0434c6d75bd4ced8d82f432a6f81d 2023-10-02T12:32:23,836 INFO [StoreOpener-29c0434c6d75bd4ced8d82f432a6f81d-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 29c0434c6d75bd4ced8d82f432a6f81d columnFamilyName f 2023-10-02T12:32:23,836 DEBUG [StoreOpener-29c0434c6d75bd4ced8d82f432a6f81d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-10-02T12:32:23,837 INFO [StoreOpener-29c0434c6d75bd4ced8d82f432a6f81d-1 {}] regionserver.HStore(324): Store=29c0434c6d75bd4ced8d82f432a6f81d/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-10-02T12:32:23,838 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/29c0434c6d75bd4ced8d82f432a6f81d 2023-10-02T12:32:23,838 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/29c0434c6d75bd4ced8d82f432a6f81d 2023-10-02T12:32:23,841 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(1076): writing seq id for 29c0434c6d75bd4ced8d82f432a6f81d 2023-10-02T12:32:23,843 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/29c0434c6d75bd4ced8d82f432a6f81d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-10-02T12:32:23,843 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(1093): Opened 29c0434c6d75bd4ced8d82f432a6f81d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11326415200, jitterRate=0.054854616522789}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-10-02T12:32:23,844 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(992): Region open journal for 29c0434c6d75bd4ced8d82f432a6f81d: 2023-10-02T12:32:23,844 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegionServer(2556): Post open deploy tasks for table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d., pid=163, masterSystemTime=1696249943828 2023-10-02T12:32:23,846 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegionServer(2583): Finished post open deploy task for table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d. 2023-10-02T12:32:23,846 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] handler.AssignRegionHandler(158): Opened table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d. 2023-10-02T12:32:23,847 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=162 updating hbase:meta row=29c0434c6d75bd4ced8d82f432a6f81d, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:23,849 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=163, resume processing ppid=162 2023-10-02T12:32:23,849 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=163, ppid=162, state=SUCCESS; OpenRegionProcedure 29c0434c6d75bd4ced8d82f432a6f81d, server=jenkins-hbase3.apache.org,38255,1696249782114 in 174 msec 2023-10-02T12:32:23,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=162, resume processing ppid=161 2023-10-02T12:32:23,851 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=162, ppid=161, state=SUCCESS; TransitRegionStateProcedure table=table1, region=29c0434c6d75bd4ced8d82f432a6f81d, ASSIGN in 332 msec 2023-10-02T12:32:23,852 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-10-02T12:32:23,852 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249943852"}]},"ts":"1696249943852"} 2023-10-02T12:32:23,853 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-10-02T12:32:23,861 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-10-02T12:32:23,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=161, state=SUCCESS; CreateTableProcedure table=table1 in 1.1820 sec 2023-10-02T12:32:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=161 2023-10-02T12:32:24,800 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:37823/backupUT/backup_1696249935939/default/test-1696249791655/archive/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1 from hbase tables test-1696249791655 to tables table1 2023-10-02T12:32:24,800 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1696249791655 into table1 2023-10-02T12:32:24,801 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-table1-1696249944800 from hdfs://localhost:37823/backupUT/backup_1696249935939/default/test-1696249791655/archive/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1 2023-10-02T12:32:24,807 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@253e1cca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:24,808 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:32:24,809 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48562, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:32:24,811 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-10-02T12:32:24,811 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-10-02T12:32:24,818 DEBUG [hconnection-0x373401fa-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:24,819 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57508, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:24,820 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-10-02T12:32:24,820 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-10-02T12:32:24,820 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/partitions_209c9214-9b0e-40b8-b255-b0e5d4c084b4 2023-10-02T12:32:25,227 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:25,228 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:25,229 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:25,229 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:26,272 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-4968781158857054511.jar 2023-10-02T12:32:26,272 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:26,273 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:26,273 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:26,274 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:26,275 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:26,275 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:26,276 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:26,276 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:26,277 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:26,277 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:32:26,278 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:32:26,279 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:32:26,279 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:32:26,280 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:32:26,281 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:32:26,281 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:32:26,282 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:32:26,282 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:32:26,283 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:32:26,283 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:32:26,284 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:32:26,284 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:26,285 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:26,286 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:26,286 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:26,286 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:26,287 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:26,288 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:32:26,288 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-10-02T12:32:26,289 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:26,289 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-10-02T12:32:26,289 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-10-02T12:32:26,290 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:32:26,303 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-10-02T12:32:26,488 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249946424/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:32:26,489 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249946424/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:32:26,550 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:37823/backupUT/backup_1696249935939/default/test-1696249791655/archive/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1/f/553dd16623454c9282013d29ef554af3_SeqId_4_ 2023-10-02T12:32:26,554 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-10-02T12:32:26,581 DEBUG [pool-535-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ce4700c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:26,583 DEBUG [hconnection-0x1c4e6826-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:26,584 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57516, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:26,590 DEBUG [pool-535-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:26,590 DEBUG [pool-535-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-10-02T12:32:26,590 DEBUG [pool-535-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase3.apache.org 2023-10-02T12:32:26,595 DEBUG [pool-535-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-table1-1696249944800/_temporary/0/_temporary/attempt_local1327539613_0025_r_000000_0/f/07e810f807824690b68a5a9db3bc26c0 is 37, key is row10/f:q1/1696249794327/Put/seqid=0 2023-10-02T12:32:26,606 WARN [Thread-2343 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase3.apache.org:38255] Specified favored nodes: [jenkins-hbase3.apache.org:38255] 2023-10-02T12:32:26,984 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2023-10-02T12:32:27,547 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-table1-1696249944800 2023-10-02T12:32:27,553 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2be0b73a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:27,555 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:27,556 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57520, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:27,563 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-table1-1696249944800/_SUCCESS 2023-10-02T12:32:27,566 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:32:27,567 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48568, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:32:27,579 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-table1-1696249944800/f/07e810f807824690b68a5a9db3bc26c0 first=Optional[row0] last=Optional[row98] 2023-10-02T12:32:27,580 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d., hostname=jenkins-hbase3.apache.org,38255,1696249782114, seqNum=2 for row with hfile group [{f,hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-table1-1696249944800/f/07e810f807824690b68a5a9db3bc26c0}] 2023-10-02T12:32:27,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-table1-1696249944800/f/07e810f807824690b68a5a9db3bc26c0 for inclusion in 29c0434c6d75bd4ced8d82f432a6f81d/f 2023-10-02T12:32:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-10-02T12:32:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(625): Region bounds: first= last= 2023-10-02T12:32:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HRegion(2520): Flush status journal for 29c0434c6d75bd4ced8d82f432a6f81d: 2023-10-02T12:32:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/hbase-staging/bulk_output-default-table1-1696249944800/f/07e810f807824690b68a5a9db3bc26c0 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__table1__h1d9e4betosbgcdebnool0s2v43drjoe5un7hck22fcldbqb738he1icdjlk6a38/f/07e810f807824690b68a5a9db3bc26c0 2023-10-02T12:32:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__table1__h1d9e4betosbgcdebnool0s2v43drjoe5un7hck22fcldbqb738he1icdjlk6a38/f/07e810f807824690b68a5a9db3bc26c0 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/29c0434c6d75bd4ced8d82f432a6f81d/f/c33a5124271849dabb59571ffe699a53_SeqId_4_ 2023-10-02T12:32:27,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e28f336, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:32:27,645 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48572, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:32:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:27,656 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57522, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:27,679 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__table1__h1d9e4betosbgcdebnool0s2v43drjoe5un7hck22fcldbqb738he1icdjlk6a38/f/07e810f807824690b68a5a9db3bc26c0 into 29c0434c6d75bd4ced8d82f432a6f81d/f as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/29c0434c6d75bd4ced8d82f432a6f81d/f/c33a5124271849dabb59571ffe699a53_SeqId_4_ - updating store file list. 2023-10-02T12:32:27,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/29c0434c6d75bd4ced8d82f432a6f81d/f/c33a5124271849dabb59571ffe699a53_SeqId_4_ into 29c0434c6d75bd4ced8d82f432a6f81d/f 2023-10-02T12:32:27,685 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__table1__h1d9e4betosbgcdebnool0s2v43drjoe5un7hck22fcldbqb738he1icdjlk6a38/f/07e810f807824690b68a5a9db3bc26c0 into 29c0434c6d75bd4ced8d82f432a6f81d/f (new location: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/29c0434c6d75bd4ced8d82f432a6f81d/f/c33a5124271849dabb59571ffe699a53_SeqId_4_) 2023-10-02T12:32:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/staging/jenkins__table1__h1d9e4betosbgcdebnool0s2v43drjoe5un7hck22fcldbqb738he1icdjlk6a38/f/07e810f807824690b68a5a9db3bc26c0 2023-10-02T12:32:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e059922, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:32:27,789 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48580, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-10-02T12:32:27,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:27,794 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57538, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-10-02T12:32:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38255 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:27,819 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:27,819 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-10-02T12:32:27,819 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-10-02T12:32:27,819 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(233): Backup: backup_1696249935939 hdfs://localhost:37823/backupUT/backup_1696249935939/default/test-1696249791655/ 2023-10-02T12:32:27,819 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-10-02T12:32:27,819 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:27,823 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$12(2664): Client=jenkins//172.31.12.81 disable table1 2023-10-02T12:32:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-10-02T12:32:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=164 2023-10-02T12:32:27,827 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249947827"}]},"ts":"1696249947827"} 2023-10-02T12:32:27,829 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-10-02T12:32:27,832 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-10-02T12:32:27,833 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=29c0434c6d75bd4ced8d82f432a6f81d, UNASSIGN}] 2023-10-02T12:32:27,833 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=165, ppid=164, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=29c0434c6d75bd4ced8d82f432a6f81d, UNASSIGN 2023-10-02T12:32:27,834 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=165 updating hbase:meta row=29c0434c6d75bd4ced8d82f432a6f81d, regionState=CLOSING, regionLocation=jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:27,836 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE; CloseRegionProcedure 29c0434c6d75bd4ced8d82f432a6f81d, server=jenkins-hbase3.apache.org,38255,1696249782114}] 2023-10-02T12:32:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=164 2023-10-02T12:32:27,989 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] handler.UnassignRegionHandler(119): Close 29c0434c6d75bd4ced8d82f432a6f81d 2023-10-02T12:32:27,989 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1668): Closing 29c0434c6d75bd4ced8d82f432a6f81d, disabling compactions & flushes 2023-10-02T12:32:27,989 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1690): Closing region table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d. 2023-10-02T12:32:27,989 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d. 2023-10-02T12:32:27,989 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1778): Acquired close lock on table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d. after waiting 0 ms 2023-10-02T12:32:27,989 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1788): Updates disabled for region table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d. 2023-10-02T12:32:27,997 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/29c0434c6d75bd4ced8d82f432a6f81d/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-10-02T12:32:27,998 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:32:27,998 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1909): Closed table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d. 2023-10-02T12:32:27,998 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1622): Region close journal for 29c0434c6d75bd4ced8d82f432a6f81d: 2023-10-02T12:32:28,000 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] handler.UnassignRegionHandler(163): Closed 29c0434c6d75bd4ced8d82f432a6f81d 2023-10-02T12:32:28,000 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=165 updating hbase:meta row=29c0434c6d75bd4ced8d82f432a6f81d, regionState=CLOSED 2023-10-02T12:32:28,011 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=166, resume processing ppid=165 2023-10-02T12:32:28,011 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=166, ppid=165, state=SUCCESS; CloseRegionProcedure 29c0434c6d75bd4ced8d82f432a6f81d, server=jenkins-hbase3.apache.org,38255,1696249782114 in 166 msec 2023-10-02T12:32:28,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=165, resume processing ppid=164 2023-10-02T12:32:28,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=165, ppid=164, state=SUCCESS; TransitRegionStateProcedure table=table1, region=29c0434c6d75bd4ced8d82f432a6f81d, UNASSIGN in 179 msec 2023-10-02T12:32:28,013 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1696249948013"}]},"ts":"1696249948013"} 2023-10-02T12:32:28,014 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-10-02T12:32:28,017 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-10-02T12:32:28,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=164, state=SUCCESS; DisableTableProcedure table=table1 in 194 msec 2023-10-02T12:32:28,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=164 2023-10-02T12:32:28,130 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster$5(2448): Client=jenkins//172.31.12.81 delete table1 2023-10-02T12:32:28,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=167, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-10-02T12:32:28,133 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=167, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-10-02T12:32:28,134 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=167, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-10-02T12:32:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=167 2023-10-02T12:32:28,136 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/29c0434c6d75bd4ced8d82f432a6f81d 2023-10-02T12:32:28,137 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/29c0434c6d75bd4ced8d82f432a6f81d/f, FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/29c0434c6d75bd4ced8d82f432a6f81d/recovered.edits] 2023-10-02T12:32:28,142 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/29c0434c6d75bd4ced8d82f432a6f81d/f/c33a5124271849dabb59571ffe699a53_SeqId_4_ to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/archive/data/default/table1/29c0434c6d75bd4ced8d82f432a6f81d/f/c33a5124271849dabb59571ffe699a53_SeqId_4_ 2023-10-02T12:32:28,145 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/29c0434c6d75bd4ced8d82f432a6f81d/recovered.edits/6.seqid to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/archive/data/default/table1/29c0434c6d75bd4ced8d82f432a6f81d/recovered.edits/6.seqid 2023-10-02T12:32:28,146 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/table1/29c0434c6d75bd4ced8d82f432a6f81d 2023-10-02T12:32:28,146 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-10-02T12:32:28,148 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=167, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-10-02T12:32:28,150 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-10-02T12:32:28,152 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-10-02T12:32:28,153 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=167, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-10-02T12:32:28,153 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-10-02T12:32:28,153 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1696249948153"}]},"ts":"9223372036854775807"} 2023-10-02T12:32:28,155 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-10-02T12:32:28,155 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 29c0434c6d75bd4ced8d82f432a6f81d, NAME => 'table1,,1696249942679.29c0434c6d75bd4ced8d82f432a6f81d.', STARTKEY => '', ENDKEY => ''}] 2023-10-02T12:32:28,155 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-10-02T12:32:28,155 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1696249948155"}]},"ts":"9223372036854775807"} 2023-10-02T12:32:28,159 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-10-02T12:32:28,161 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(127): Finished pid=167, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-10-02T12:32:28,162 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=167, state=SUCCESS; DeleteTableProcedure table=table1 in 30 msec 2023-10-02T12:32:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=167 2023-10-02T12:32:28,253 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleCommand Thread=604 (was 604), OpenFileDescriptor=690 (was 697), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=515 (was 503) - SystemLoadAverage LEAK? -, ProcessCount=181 (was 181), AvailableMemoryMB=1764 (was 2108) 2023-10-02T12:32:28,253 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-10-02T12:32:28,271 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleDNE Thread=604, OpenFileDescriptor=690, MaxFileDescriptor=60000, SystemLoadAverage=515, ProcessCount=181, AvailableMemoryMB=1764 2023-10-02T12:32:28,271 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-10-02T12:32:28,271 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] backup.TestFullRestore(302): test restore fails on multiple tables that do not exist 2023-10-02T12:32:28,275 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@271438c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:28,282 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:28,283 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57544, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:28,285 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-10-02T12:32:28,286 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:48584, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-10-02T12:32:28,297 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-10-02T12:32:28,302 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:32:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249948302 (current time:1696249948302). 2023-10-02T12:32:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:32:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-10-02T12:32:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e3bb7fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:28,311 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57560, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:32:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=168, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-10-02T12:32:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 168 2023-10-02T12:32:28,316 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:32:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=168 2023-10-02T12:32:28,317 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:32:28,328 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor129.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:32:28,331 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:32:28,340 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:32:28,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987}] 2023-10-02T12:32:28,342 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=169, ppid=168, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:32:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=168 2023-10-02T12:32:28,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=169 2023-10-02T12:32:28,494 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:32:28,494 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegion(2819): Flushing 7ec576d712d0f3b0c0592584b88ce987 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-10-02T12:32:28,498 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/7e997c55a6b941feba5edf35d1655e75 is 169, key is trslm:hdfs://localhost:37823/backupUT\x00test-1696249791655/meta:log-roll-map/1696249942207/Put/seqid=0 2023-10-02T12:32:28,503 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/7e997c55a6b941feba5edf35d1655e75 2023-10-02T12:32:28,511 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/7ac5c97a6c3846beb6f444e8a7269d64 is 452, key is session:backup_1696249935939/session:context/1696249942630/Put/seqid=0 2023-10-02T12:32:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=168 2023-10-02T12:32:28,916 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/7ac5c97a6c3846beb6f444e8a7269d64 2023-10-02T12:32:28,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=168 2023-10-02T12:32:28,923 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/7e997c55a6b941feba5edf35d1655e75 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/7e997c55a6b941feba5edf35d1655e75 2023-10-02T12:32:28,927 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/7e997c55a6b941feba5edf35d1655e75, entries=4, sequenceid=138, filesize=5.5 K 2023-10-02T12:32:28,928 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/7ac5c97a6c3846beb6f444e8a7269d64 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/7ac5c97a6c3846beb6f444e8a7269d64 2023-10-02T12:32:28,933 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/7ac5c97a6c3846beb6f444e8a7269d64, entries=2, sequenceid=138, filesize=5.4 K 2023-10-02T12:32:28,934 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 7ec576d712d0f3b0c0592584b88ce987 in 440ms, sequenceid=138, compaction requested=true 2023-10-02T12:32:28,934 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegion(2520): Flush status journal for 7ec576d712d0f3b0c0592584b88ce987: 2023-10-02T12:32:28,934 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. for snapshot_backup_system completed. 2023-10-02T12:32:28,934 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987.' region-info for snapshot=snapshot_backup_system 2023-10-02T12:32:28,934 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:32:28,934 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/85d2568c07cc4be686b8e706b89254ce, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/9bf0c2ed3e5c43ba8bb2287a7e2731fa, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b4a8e50177b941698e5089bffe91b251, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/23e51484216f4e7a9dca73d1bb9d8f09, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/f1ceb39b31074c03be29750a860f2cc8, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/d0f3dcf500344358956a19104aad9e8f, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/af19b5a70b5a4fe19a14d9bf11f1ca14, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/7e997c55a6b941feba5edf35d1655e75] hfiles 2023-10-02T12:32:28,934 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (1/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/be4f942a12804891a1f8bc2a12951f98 for snapshot=snapshot_backup_system 2023-10-02T12:32:28,934 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (2/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/eeec1eb03cf84adab876e360f800c255 for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (3/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b9b47a46600b4a0483abe90aca8688e8 for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (4/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/85d2568c07cc4be686b8e706b89254ce for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (5/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/e5edb6dc66aa4ea2b538ff730ff1a4e5 for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (6/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/9bf0c2ed3e5c43ba8bb2287a7e2731fa for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (7/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/b4a8e50177b941698e5089bffe91b251 for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (8/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/23e51484216f4e7a9dca73d1bb9d8f09 for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (9/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/f1ceb39b31074c03be29750a860f2cc8 for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (10/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/d0f3dcf500344358956a19104aad9e8f for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (11/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/af19b5a70b5a4fe19a14d9bf11f1ca14 for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (12/12): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/7e997c55a6b941feba5edf35d1655e75 for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/f8db732eb5cd417eaa236f25fd437245, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/fe4c59a0e2584ff5980ec23dd90e1c0b, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/2129c391bd7447f0b05fea1084f2d47d, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/120da2f382584d43adc4417dfe387fae, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/75d622dd3189402a9723a0642f3437f9, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/56ef49f7bed44dd1b83792cad64956f0, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/eac7dd6dd5764b7f91b006a5dc2e0b23, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/6eccbee667ca41b9851e81d99bcb3699, hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/7ac5c97a6c3846beb6f444e8a7269d64] hfiles 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (1/13): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/542bcb858b3b482dbbc3e2620ea1c1c0 for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (2/13): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/96b7c481d10942d6a12bd6cb944c1f6c for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (3/13): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/5fb18f8bc21a4dc1b98c0af588fef229 for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (4/13): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/d30d36b7102e4495bb9961fcaa5fc423 for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (5/13): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/f8db732eb5cd417eaa236f25fd437245 for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (6/13): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/fe4c59a0e2584ff5980ec23dd90e1c0b for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (7/13): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/2129c391bd7447f0b05fea1084f2d47d for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (8/13): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/120da2f382584d43adc4417dfe387fae for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (9/13): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/75d622dd3189402a9723a0642f3437f9 for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (10/13): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/56ef49f7bed44dd1b83792cad64956f0 for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (11/13): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/eac7dd6dd5764b7f91b006a5dc2e0b23 for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (12/13): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/6eccbee667ca41b9851e81d99bcb3699 for snapshot=snapshot_backup_system 2023-10-02T12:32:28,935 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (13/13): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/7ac5c97a6c3846beb6f444e8a7269d64 for snapshot=snapshot_backup_system 2023-10-02T12:32:29,342 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:32:29,342 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=169 2023-10-02T12:32:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=169 2023-10-02T12:32:29,342 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:32:29,344 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=169, ppid=168, state=RUNNABLE; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:32:29,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=169, resume processing ppid=168 2023-10-02T12:32:29,346 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:32:29,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=169, ppid=168, state=SUCCESS; SnapshotRegionProcedure 7ec576d712d0f3b0c0592584b88ce987 in 1.0050 sec 2023-10-02T12:32:29,346 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:32:29,347 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:32:29,347 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-10-02T12:32:29,348 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system 2023-10-02T12:32:29,362 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:32:29,379 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-10-02T12:32:29,382 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:32:29,382 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_backup_system 2023-10-02T12:32:29,384 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:32:29,384 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 168 2023-10-02T12:32:29,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=168, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.0720 sec 2023-10-02T12:32:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=168 2023-10-02T12:32:29,422 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(115): Backup backup_1696249948275 started at 1696249949422. 2023-10-02T12:32:29,424 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(119): Backup session backup_1696249948275 has been started. 2023-10-02T12:32:29,428 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-10-02T12:32:29,429 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(948): Client=jenkins//172.31.12.81 procedure request for: rolllog-proc 2023-10-02T12:32:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-10-02T12:32:29,429 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-10-02T12:32:29,429 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-10-02T12:32:29,430 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-10-02T12:32:29,430 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-10-02T12:32:29,431 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:32:29,432 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:32:29,432 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:29,432 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:32:29,433 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-10-02T12:32:29,433 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-10-02T12:32:29,434 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-10-02T12:32:29,434 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-10-02T12:32:29,434 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-10-02T12:32:29,434 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-10-02T12:32:29,434 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-10-02T12:32:29,434 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-10-02T12:32:29,434 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-10-02T12:32:29,435 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-10-02T12:32:29,435 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-10-02T12:32:29,435 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure (rolllog) in zk 2023-10-02T12:32:29,437 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-10-02T12:32:29,438 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:29,438 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:32:29,438 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:32:29,438 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-10-02T12:32:29,438 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:32:29,438 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:32:29,439 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:32:29,439 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:29,439 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:32:29,440 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase3.apache.org,38255,1696249782114' joining acquired barrier for procedure 'rolllog' on coordinator 2023-10-02T12:32:29,440 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@ac933c6[Count = 0] remaining members to acquire global barrier 2023-10-02T12:32:29,440 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-10-02T12:32:29,440 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-10-02T12:32:29,441 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-10-02T12:32:29,441 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-10-02T12:32:29,441 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-10-02T12:32:29,441 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-10-02T12:32:29,441 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-10-02T12:32:29,442 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:29,442 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1696249938173 highest: 1696249938173 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:29,442 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114:(num 1696249938173) roll requested 2023-10-02T12:32:29,455 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249949442, exclude list is [], retry=0 2023-10-02T12:32:29,457 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:32:29,458 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249938173 with entries=13, filesize=3.13 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249949442 2023-10-02T12:32:29,458 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:32:29,458 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249938173 is not closed yet, will try archiving it next time 2023-10-02T12:32:29,459 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta:.meta(num 1696249938190) roll requested 2023-10-02T12:32:29,459 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249932323 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/oldWALs/jenkins-hbase3.apache.org%2C38255%2C1696249782114.1696249932323 2023-10-02T12:32:29,472 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249949459.meta, exclude list is [], retry=0 2023-10-02T12:32:29,475 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK] 2023-10-02T12:32:29,476 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249938190.meta with entries=12, filesize=2.74 KB; new WAL /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249949459.meta 2023-10-02T12:32:29,476 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:34237,DS-d510eea1-3a43-45d6-b828-f5fc721ec0c7,DISK]] 2023-10-02T12:32:29,476 DEBUG [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249938190.meta is not closed yet, will try archiving it next time 2023-10-02T12:32:29,482 DEBUG [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-10-02T12:32:29,482 INFO [rs(jenkins-hbase3.apache.org,38255,1696249782114)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1696249949442 on jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:29,489 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-10-02T12:32:29,489 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase3.apache.org,38255,1696249782114' in zk 2023-10-02T12:32:29,493 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-10-02T12:32:29,493 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:32:29,493 DEBUG [member: 'jenkins-hbase3.apache.org,38255,1696249782114' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-10-02T12:32:29,493 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:29,494 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:32:29,494 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:32:29,494 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:32:29,494 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:32:29,495 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:32:29,495 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:29,495 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:32:29,495 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:32:29,496 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:29,497 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase3.apache.org,38255,1696249782114': 2023-10-02T12:32:29,497 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase3.apache.org,38255,1696249782114' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-10-02T12:32:29,497 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-10-02T12:32:29,497 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-10-02T12:32:29,497 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-10-02T12:32:29,497 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-10-02T12:32:29,497 INFO [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-10-02T12:32:29,499 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-10-02T12:32:29,499 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-10-02T12:32:29,499 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:32:29,499 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-10-02T12:32:29,499 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-10-02T12:32:29,499 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:32:29,500 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:32:29,500 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-10-02T12:32:29,500 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-10-02T12:32:29,500 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:32:29,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-10-02T12:32:29,501 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:32:29,502 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:29,502 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-10-02T12:32:29,503 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-10-02T12:32:29,503 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:29,508 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-10-02T12:32:29,508 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-10-02T12:32:29,508 DEBUG [(jenkins-hbase3.apache.org,40619,1696249778965)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-10-02T12:32:29,508 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-10-02T12:32:29,508 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-10-02T12:32:29,509 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-10-02T12:32:29,509 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-10-02T12:32:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1249): Checking to see if procedure from request:rolllog-proc is done 2023-10-02T12:32:29,615 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249949614_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } 2023-10-02T12:32:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249949615 (current time:1696249949615). 2023-10-02T12:32:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:32:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249949614_ns2_test-16962497916551 VERSION not specified, setting to 2 2023-10-02T12:32:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d7e6208, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:29,629 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:57572, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:32:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=170, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1696249949614_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } 2023-10-02T12:32:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249949614_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 }, snapshot procedure id = 170 2023-10-02T12:32:29,637 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1696249949614_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:32:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=170 2023-10-02T12:32:29,638 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1696249949614_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:32:29,641 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249949614_ns2_test-16962497916551 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor129.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:32:29,644 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1696249949614_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:32:29,652 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1696249949614_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:32:29,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE; SnapshotRegionProcedure 1ff8cee0e4d9c23004d17b9d828c2c96}] 2023-10-02T12:32:29,654 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=171, ppid=170, state=RUNNABLE; SnapshotRegionProcedure 1ff8cee0e4d9c23004d17b9d828c2c96 2023-10-02T12:32:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=170 2023-10-02T12:32:29,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=171 2023-10-02T12:32:29,805 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96. 2023-10-02T12:32:29,806 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.HRegion(2520): Flush status journal for 1ff8cee0e4d9c23004d17b9d828c2c96: 2023-10-02T12:32:29,806 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96. for snapshot_1696249949614_ns2_test-16962497916551 completed. 2023-10-02T12:32:29,806 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(240): Storing 'ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96.' region-info for snapshot=snapshot_1696249949614_ns2_test-16962497916551 2023-10-02T12:32:29,806 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:32:29,806 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96/f/eec10f547780452b96a08ad7a8ce56ca_SeqId_4_] hfiles 2023-10-02T12:32:29,806 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96/f/eec10f547780452b96a08ad7a8ce56ca_SeqId_4_ for snapshot=snapshot_1696249949614_ns2_test-16962497916551 2023-10-02T12:32:29,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=170 2023-10-02T12:32:30,212 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96. 2023-10-02T12:32:30,212 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=171 2023-10-02T12:32:30,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=171 2023-10-02T12:32:30,212 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249949614_ns2_test-16962497916551 on region 1ff8cee0e4d9c23004d17b9d828c2c96 2023-10-02T12:32:30,214 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=171, ppid=170, state=RUNNABLE; SnapshotRegionProcedure 1ff8cee0e4d9c23004d17b9d828c2c96 2023-10-02T12:32:30,216 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=171, resume processing ppid=170 2023-10-02T12:32:30,216 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1696249949614_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:32:30,216 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=171, ppid=170, state=SUCCESS; SnapshotRegionProcedure 1ff8cee0e4d9c23004d17b9d828c2c96 in 563 msec 2023-10-02T12:32:30,217 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1696249949614_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:32:30,217 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1696249949614_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:32:30,217 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249949614_ns2_test-16962497916551 2023-10-02T12:32:30,218 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249949614_ns2_test-16962497916551 2023-10-02T12:32:30,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=170 2023-10-02T12:32:30,626 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1696249949614_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:32:30,631 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1696249949614_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:32:30,632 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249949614_ns2_test-16962497916551 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249949614_ns2_test-16962497916551 2023-10-02T12:32:30,632 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1696249949614_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:32:30,633 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249949614_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 }, snapshot procedure id = 170 2023-10-02T12:32:30,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=170, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1696249949614_ns2_test-16962497916551 table=ns2:test-16962497916551 type=FLUSH ttl=0 } in 999 msec 2023-10-02T12:32:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=170 2023-10-02T12:32:30,752 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1687): Client=jenkins//172.31.12.81 snapshot request for:{ ss=snapshot_1696249950751_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } 2023-10-02T12:32:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1696249950752 (current time:1696249950752). 2023-10-02T12:32:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-10-02T12:32:30,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1696249950751_ns3_test-16962497916552 VERSION not specified, setting to 2 2023-10-02T12:32:30,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fa071d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-10-02T12:32:30,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-10-02T12:32:30,795 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(477): Connection from 172.31.12.81:50244, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-10-02T12:32:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-10-02T12:32:30,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure2.ProcedureExecutor(1032): Stored pid=172, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1696249950751_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } 2023-10-02T12:32:30,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1696249950751_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 }, snapshot procedure id = 172 2023-10-02T12:32:30,800 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1696249950751_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-10-02T12:32:30,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=172 2023-10-02T12:32:30,801 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1696249950751_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-10-02T12:32:30,804 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot, current working dir path: hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249950751_ns3_test-16962497916552 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor129.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-10-02T12:32:30,807 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1696249950751_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-10-02T12:32:30,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=172 2023-10-02T12:32:31,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=172 2023-10-02T12:32:31,214 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1696249950751_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-10-02T12:32:31,214 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE; SnapshotRegionProcedure 8f9314653c5767b76ceff134e893c29a}] 2023-10-02T12:32:31,216 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=173, ppid=172, state=RUNNABLE; SnapshotRegionProcedure 8f9314653c5767b76ceff134e893c29a 2023-10-02T12:32:31,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38255 {}] regionserver.RSRpcServices(3924): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=173 2023-10-02T12:32:31,368 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a. 2023-10-02T12:32:31,368 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.HRegion(2520): Flush status journal for 8f9314653c5767b76ceff134e893c29a: 2023-10-02T12:32:31,368 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a. for snapshot_1696249950751_ns3_test-16962497916552 completed. 2023-10-02T12:32:31,368 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] snapshot.SnapshotManifest(240): Storing 'ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a.' region-info for snapshot=snapshot_1696249950751_ns3_test-16962497916552 2023-10-02T12:32:31,368 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-10-02T12:32:31,368 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-10-02T12:32:31,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=172 2023-10-02T12:32:31,774 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a. 2023-10-02T12:32:31,774 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase3:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=173 2023-10-02T12:32:31,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.HMaster(3995): Remote procedure done, pid=173 2023-10-02T12:32:31,774 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1696249950751_ns3_test-16962497916552 on region 8f9314653c5767b76ceff134e893c29a 2023-10-02T12:32:31,776 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=173, ppid=172, state=RUNNABLE; SnapshotRegionProcedure 8f9314653c5767b76ceff134e893c29a 2023-10-02T12:32:31,778 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=173, resume processing ppid=172 2023-10-02T12:32:31,778 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=173, ppid=172, state=SUCCESS; SnapshotRegionProcedure 8f9314653c5767b76ceff134e893c29a in 563 msec 2023-10-02T12:32:31,778 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1696249950751_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-10-02T12:32:31,779 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1696249950751_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-10-02T12:32:31,779 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1696249950751_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-10-02T12:32:31,779 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1696249950751_ns3_test-16962497916552 2023-10-02T12:32:31,780 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249950751_ns3_test-16962497916552 2023-10-02T12:32:31,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=172 2023-10-02T12:32:32,190 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1696249950751_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-10-02T12:32:32,194 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1696249950751_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-10-02T12:32:32,194 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/.tmp/snapshot_1696249950751_ns3_test-16962497916552 to hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249950751_ns3_test-16962497916552 2023-10-02T12:32:32,195 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1696249950751_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-10-02T12:32:32,195 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1696249950751_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 }, snapshot procedure id = 172 2023-10-02T12:32:32,196 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=172, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1696249950751_ns3_test-16962497916552 table=ns3:test-16962497916552 type=FLUSH ttl=0 } in 1.3980 sec 2023-10-02T12:32:32,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(1290): Checking to see if procedure is done pid=172 2023-10-02T12:32:32,907 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1696249948275 2023-10-02T12:32:32,907 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-10-02T12:32:32,907 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-10-02T12:32:32,907 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249948275_ns2:test-16962497916551 2023-10-02T12:32:32,907 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249949614_ns2_test-16962497916551 to hdfs://localhost:37823/backupUT/backup_1696249948275/ns2/test-16962497916551/ 2023-10-02T12:32:32,908 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:32:32,930 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:32:32,931 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,40619,1696249778965_-1483518392_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37823/backupUT/backup_1696249948275/ns2/test-16962497916551, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37823/backupUT/backup_1696249948275/ns2/test-16962497916551/.hbase-snapshot/.tmp/snapshot_1696249949614_ns2_test-16962497916551 2023-10-02T12:32:32,931 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07, snapshotDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249949614_ns2_test-16962497916551. 2023-10-02T12:32:32,938 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249949614_ns2_test-16962497916551 to hdfs://localhost:37823/backupUT/backup_1696249948275/ns2/test-16962497916551/.hbase-snapshot/.tmp/snapshot_1696249949614_ns2_test-16962497916551 2023-10-02T12:32:33,350 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:33,350 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:33,351 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:33,351 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:34,346 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-2369560707516211294.jar 2023-10-02T12:32:34,346 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:34,346 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:34,347 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:34,347 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:34,347 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:34,348 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:34,348 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:34,348 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:34,349 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:34,349 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:32:34,350 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:32:34,350 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:32:34,351 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:32:34,351 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:32:34,351 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:32:34,352 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:32:34,352 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:32:34,352 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:32:34,353 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:32:34,353 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:32:34,353 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:32:34,354 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:32:34,354 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:32:34,355 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:32:34,355 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:32:34,355 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:32:34,356 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:32:34,356 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:32:34,399 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249949614_ns2_test-16962497916551' hfile list 2023-10-02T12:32:34,400 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-10-02T12:32:34,471 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249954435/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:32:34,471 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249954435/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:32:34,523 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-10-02T12:32:34,547 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96/f/eec10f547780452b96a08ad7a8ce56ca_SeqId_4_ output=hdfs://localhost:37823/backupUT/backup_1696249948275/ns2/test-16962497916551/archive/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96/f/eec10f547780452b96a08ad7a8ce56ca_SeqId_4_ 2023-10-02T12:32:34,547 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 8.321M/sec 2023-10-02T12:32:35,501 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:32:35,502 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:32:35,508 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249949614_ns2_test-16962497916551 2023-10-02T12:32:35,508 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249949614_ns2_test-16962497916551 finished. 2023-10-02T12:32:35,508 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1696249948275_ns3:test-16962497916552 2023-10-02T12:32:35,508 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1696249950751_ns3_test-16962497916552 to hdfs://localhost:37823/backupUT/backup_1696249948275/ns3/test-16962497916552/ 2023-10-02T12:32:35,508 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-10-02T12:32:35,531 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07 2023-10-02T12:32:35,531 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase3.apache.org,40619,1696249778965_-564947212_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:37823/backupUT/backup_1696249948275/ns3/test-16962497916552, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:37823/backupUT/backup_1696249948275/ns3/test-16962497916552/.hbase-snapshot/.tmp/snapshot_1696249950751_ns3_test-16962497916552 2023-10-02T12:32:35,531 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:37823, inputRoot=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07, snapshotDir=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249950751_ns3_test-16962497916552. 2023-10-02T12:32:35,537 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/.hbase-snapshot/snapshot_1696249950751_ns3_test-16962497916552 to hdfs://localhost:37823/backupUT/backup_1696249948275/ns3/test-16962497916552/.hbase-snapshot/.tmp/snapshot_1696249950751_ns3_test-16962497916552 2023-10-02T12:32:35,957 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:35,957 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:35,957 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:35,958 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:37,015 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop-5793678523897131578.jar 2023-10-02T12:32:37,015 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:37,015 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:37,016 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:37,016 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:37,017 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:37,017 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:37,017 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:37,018 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:37,018 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-10-02T12:32:37,019 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.4/hbase-shaded-miscellaneous-4.1.4.jar 2023-10-02T12:32:37,020 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.4/hbase-shaded-gson-4.1.4.jar 2023-10-02T12:32:37,021 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.4/hbase-shaded-protobuf-4.1.4.jar 2023-10-02T12:32:37,021 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.4/hbase-shaded-netty-4.1.4.jar 2023-10-02T12:32:37,022 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.4/hbase-unsafe-4.1.4.jar 2023-10-02T12:32:37,022 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar 2023-10-02T12:32:37,022 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-10-02T12:32:37,023 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-10-02T12:32:37,023 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-10-02T12:32:37,024 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-10-02T12:32:37,024 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-10-02T12:32:37,025 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-10-02T12:32:37,025 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:32:37,026 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:32:37,026 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:32:37,027 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:32:37,027 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-10-02T12:32:37,028 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:32:37,028 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-10-02T12:32:37,078 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1696249950751_ns3_test-16962497916552' hfile list 2023-10-02T12:32:37,157 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249957118/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-10-02T12:32:37,157 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/mapreduce.cluster.local.dir/1696249957118/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-10-02T12:32:38,195 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-10-02T12:32:38,196 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-10-02T12:32:38,201 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1696249950751_ns3_test-16962497916552 2023-10-02T12:32:38,202 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1696249950751_ns3_test-16962497916552 finished. 2023-10-02T12:32:38,202 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1034): ns2:test-16962497916551 2023-10-02T12:32:38,202 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1034): ns3:test-16962497916552 2023-10-02T12:32:38,209 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249948275 2023-10-02T12:32:38,209 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:32:38,216 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37823/backupUT/backup_1696249948275/.backup.manifest 2023-10-02T12:32:38,216 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1696249948275 2023-10-02T12:32:38,216 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-10-02T12:32:38,240 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:37823/backupUT/backup_1696249948275/.backup.manifest 2023-10-02T12:32:38,240 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(358): Backup backup_1696249948275 finished: type=FULL,tablelist=ns2:test-16962497916551;ns3:test-16962497916552,targetRootDir=hdfs://localhost:37823/backupUT,startts=1696249949422,completets=1696249958209,bytescopied=0 2023-10-02T12:32:38,240 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-10-02T12:32:38,240 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249949614_ns2_test-16962497916551 2023-10-02T12:32:38,241 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249949614_ns2_test-16962497916551" 2023-10-02T12:32:38,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249949614_ns2_test-16962497916551 2023-10-02T12:32:38,244 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249949614_ns2_test-16962497916551 for backup backup_1696249948275 succeeded. 2023-10-02T12:32:38,245 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1696249950751_ns3_test-16962497916552 2023-10-02T12:32:38,245 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_1696249950751_ns3_test-16962497916552" 2023-10-02T12:32:38,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1696249950751_ns3_test-16962497916552 2023-10-02T12:32:38,248 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1696249950751_ns3_test-16962497916552 for backup backup_1696249948275 succeeded. 2023-10-02T12:32:38,249 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-10-02T12:32:38,253 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] master.MasterRpcServices(761): Client=jenkins//172.31.12.81 delete name: "snapshot_backup_system" 2023-10-02T12:32:38,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40619 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-10-02T12:32:38,257 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-10-02T12:32:38,258 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-10-02T12:32:38,264 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.TableBackupClient(376): Backup backup_1696249948275 completed. 2023-10-02T12:32:38,264 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:38,275 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(101): HBase table ns2:table2 does not exist. It will be created during restore process 2023-10-02T12:32:38,275 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(101): HBase table ns3:table3_restore does not exist. It will be created during restore process 2023-10-02T12:32:38,276 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249948275/.backup.manifest 2023-10-02T12:32:38,278 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249948275/.backup.manifest 2023-10-02T12:32:38,279 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249948275/.backup.manifest 2023-10-02T12:32:38,281 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249948275/.backup.manifest 2023-10-02T12:32:38,281 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:37823/backupUT/backup_1696249948275/.backup.manifest 2023-10-02T12:32:38,283 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1696249948275/.backup.manifest 2023-10-02T12:32:38,283 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] impl.RestoreTablesClient(148): Restoring 'faketable1' to 'ns2:table2' from full backup image hdfs://localhost:37823/backupUT/backup_1696249948275/default/faketable1 2023-10-02T12:32:38,304 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleDNE Thread=603 (was 604), OpenFileDescriptor=694 (was 690) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=467 (was 515), ProcessCount=181 (was 181), AvailableMemoryMB=1564 (was 1764) 2023-10-02T12:32:38,304 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.ResourceChecker(130): Thread=603 is superior to 500 2023-10-02T12:32:38,312 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-10-02T12:32:38,313 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:38,313 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-10-02T12:32:38,313 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] util.JVMClusterUtil(257): Found active master hash=2132195928, stopped=false 2023-10-02T12:32:38,313 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] master.ServerManager(910): Cluster shutdown requested of master=jenkins-hbase3.apache.org,40619,1696249778965 2023-10-02T12:32:38,317 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] procedure2.ProcedureExecutor(630): Stopping 2023-10-02T12:32:38,317 DEBUG [Listener at jenkins-hbase3.apache.org/38049 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:38,317 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,38255,1696249782114' ***** 2023-10-02T12:32:38,317 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-10-02T12:32:38,318 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-10-02T12:32:38,324 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-10-02T12:32:38,333 INFO [regionserver/jenkins-hbase3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-10-02T12:32:38,334 INFO [RS:0;jenkins-hbase3:38255 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@1261a172{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-10-02T12:32:38,338 INFO [RS:0;jenkins-hbase3:38255 {}] server.AbstractConnector(383): Stopped ServerConnector@5d09fa3f{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:32:38,339 INFO [RS:0;jenkins-hbase3:38255 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-10-02T12:32:38,340 INFO [RS:0;jenkins-hbase3:38255 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@ae5dddd{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-10-02T12:32:38,340 INFO [RS:0;jenkins-hbase3:38255 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@71013728{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop.log.dir/,STOPPED} 2023-10-02T12:32:38,343 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.HeapMemoryManager(220): Stopping 2023-10-02T12:32:38,343 INFO [RS:0;jenkins-hbase3:38255 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-10-02T12:32:38,343 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-10-02T12:32:38,343 INFO [RS:0;jenkins-hbase3:38255 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-10-02T12:32:38,343 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(3528): Received CLOSE for 5ff54f3093326c5aa5ae7ad49096dbf7 2023-10-02T12:32:38,344 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(3528): Received CLOSE for 7ec576d712d0f3b0c0592584b88ce987 2023-10-02T12:32:38,344 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(3528): Received CLOSE for 286cb7fdcd7cb6dd93ac8373e6047f98 2023-10-02T12:32:38,344 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(3528): Received CLOSE for 8f9314653c5767b76ceff134e893c29a 2023-10-02T12:32:38,344 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(3528): Received CLOSE for d30d346a1595b3fd84f804f2773895cc 2023-10-02T12:32:38,344 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(3528): Received CLOSE for 1ff8cee0e4d9c23004d17b9d828c2c96 2023-10-02T12:32:38,344 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(3528): Received CLOSE for 35dcfa6f2558560536edbd70539305b1 2023-10-02T12:32:38,344 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,38255,1696249782114 2023-10-02T12:32:38,344 DEBUG [RS:0;jenkins-hbase3:38255 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:38,345 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 5ff54f3093326c5aa5ae7ad49096dbf7, disabling compactions & flushes 2023-10-02T12:32:38,345 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7. 2023-10-02T12:32:38,345 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7. 2023-10-02T12:32:38,345 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-10-02T12:32:38,345 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7. after waiting 0 ms 2023-10-02T12:32:38,345 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-10-02T12:32:38,345 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7. 2023-10-02T12:32:38,345 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-10-02T12:32:38,345 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-10-02T12:32:38,345 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-10-02T12:32:38,345 DEBUG [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(1583): Online Regions={5ff54f3093326c5aa5ae7ad49096dbf7=ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7., 7ec576d712d0f3b0c0592584b88ce987=backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987., 286cb7fdcd7cb6dd93ac8373e6047f98=backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98., 1588230740=hbase:meta,,1.1588230740, 8f9314653c5767b76ceff134e893c29a=ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a., d30d346a1595b3fd84f804f2773895cc=hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc., 1ff8cee0e4d9c23004d17b9d828c2c96=ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96., 35dcfa6f2558560536edbd70539305b1=test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1.} 2023-10-02T12:32:38,348 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-10-02T12:32:38,348 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-10-02T12:32:38,348 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-10-02T12:32:38,348 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-10-02T12:32:38,348 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-10-02T12:32:38,348 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=38.00 KB heapSize=66.30 KB 2023-10-02T12:32:38,348 DEBUG [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 1ff8cee0e4d9c23004d17b9d828c2c96, 286cb7fdcd7cb6dd93ac8373e6047f98, 35dcfa6f2558560536edbd70539305b1, 5ff54f3093326c5aa5ae7ad49096dbf7, 7ec576d712d0f3b0c0592584b88ce987, 8f9314653c5767b76ceff134e893c29a, d30d346a1595b3fd84f804f2773895cc 2023-10-02T12:32:38,354 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns4/test-16962497916553/5ff54f3093326c5aa5ae7ad49096dbf7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-10-02T12:32:38,354 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:32:38,355 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7. 2023-10-02T12:32:38,355 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 5ff54f3093326c5aa5ae7ad49096dbf7: 2023-10-02T12:32:38,355 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-16962497916553,,1696249797750.5ff54f3093326c5aa5ae7ad49096dbf7. 2023-10-02T12:32:38,356 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 7ec576d712d0f3b0c0592584b88ce987, disabling compactions & flushes 2023-10-02T12:32:38,356 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:32:38,356 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:32:38,356 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. after waiting 0 ms 2023-10-02T12:32:38,356 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:32:38,356 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 7ec576d712d0f3b0c0592584b88ce987 2/2 column families, dataSize=1.46 KB heapSize=2.61 KB 2023-10-02T12:32:38,360 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/8327a581f08447bfa9746b0a45ba018f is 171, key is trslm:hdfs://localhost:37823/backupUT\x00ns2:test-16962497916551/meta:log-roll-map/1696249958204/Put/seqid=0 2023-10-02T12:32:38,374 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/8327a581f08447bfa9746b0a45ba018f 2023-10-02T12:32:38,390 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740/.tmp/info/d64136d3833e462daef3554757f9b4b2 is 159, key is ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96./info:regioninfo/1696249906710/Put/seqid=0 2023-10-02T12:32:38,390 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/2e2069b87a274017a4347dbceea067b6 is 524, key is session:backup_1696249948275/session:context/1696249958257/Put/seqid=0 2023-10-02T12:32:38,400 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=775 B at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/2e2069b87a274017a4347dbceea067b6 2023-10-02T12:32:38,406 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/meta/8327a581f08447bfa9746b0a45ba018f as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/8327a581f08447bfa9746b0a45ba018f 2023-10-02T12:32:38,413 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/meta/8327a581f08447bfa9746b0a45ba018f, entries=6, sequenceid=148, filesize=5.8 K 2023-10-02T12:32:38,418 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/.tmp/session/2e2069b87a274017a4347dbceea067b6 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/2e2069b87a274017a4347dbceea067b6 2023-10-02T12:32:38,424 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/session/2e2069b87a274017a4347dbceea067b6, entries=2, sequenceid=148, filesize=5.5 K 2023-10-02T12:32:38,425 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~1.46 KB/1492, heapSize ~2.58 KB/2640, currentSize=0 B/0 for 7ec576d712d0f3b0c0592584b88ce987 in 69ms, sequenceid=148, compaction requested=true 2023-10-02T12:32:38,469 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system/7ec576d712d0f3b0c0592584b88ce987/recovered.edits/151.seqid, newMaxSeqId=151, maxSeqId=1 2023-10-02T12:32:38,470 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:32:38,471 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:32:38,471 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 7ec576d712d0f3b0c0592584b88ce987: 2023-10-02T12:32:38,471 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1696249798592.7ec576d712d0f3b0c0592584b88ce987. 2023-10-02T12:32:38,472 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 286cb7fdcd7cb6dd93ac8373e6047f98, disabling compactions & flushes 2023-10-02T12:32:38,473 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98. 2023-10-02T12:32:38,473 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98. 2023-10-02T12:32:38,473 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98. after waiting 0 ms 2023-10-02T12:32:38,473 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98. 2023-10-02T12:32:38,473 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 286cb7fdcd7cb6dd93ac8373e6047f98 2/2 column families, dataSize=5.66 KB heapSize=8.50 KB 2023-10-02T12:32:38,493 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system_bulk/286cb7fdcd7cb6dd93ac8373e6047f98/.tmp/meta/dca86db70b4e44cebc68e3ffb7115a16 is 323, key is bulk:ns2:test-16962497916551:1ff8cee0e4d9c23004d17b9d828c2c96:eec10f547780452b96a08ad7a8ce56ca_SeqId_4_/meta:path/1696249910091/Put/seqid=0 2023-10-02T12:32:38,502 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.66 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system_bulk/286cb7fdcd7cb6dd93ac8373e6047f98/.tmp/meta/dca86db70b4e44cebc68e3ffb7115a16 2023-10-02T12:32:38,510 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system_bulk/286cb7fdcd7cb6dd93ac8373e6047f98/.tmp/meta/dca86db70b4e44cebc68e3ffb7115a16 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system_bulk/286cb7fdcd7cb6dd93ac8373e6047f98/meta/dca86db70b4e44cebc68e3ffb7115a16 2023-10-02T12:32:38,518 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system_bulk/286cb7fdcd7cb6dd93ac8373e6047f98/meta/dca86db70b4e44cebc68e3ffb7115a16, entries=16, sequenceid=12, filesize=8.1 K 2023-10-02T12:32:38,519 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~5.66 KB/5800, heapSize ~8.23 KB/8432, currentSize=0 B/0 for 286cb7fdcd7cb6dd93ac8373e6047f98 in 46ms, sequenceid=12, compaction requested=false 2023-10-02T12:32:38,539 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/backup/system_bulk/286cb7fdcd7cb6dd93ac8373e6047f98/recovered.edits/15.seqid, newMaxSeqId=15, maxSeqId=1 2023-10-02T12:32:38,540 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:32:38,541 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98. 2023-10-02T12:32:38,541 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 286cb7fdcd7cb6dd93ac8373e6047f98: 2023-10-02T12:32:38,541 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1696249799722.286cb7fdcd7cb6dd93ac8373e6047f98. 2023-10-02T12:32:38,541 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 8f9314653c5767b76ceff134e893c29a, disabling compactions & flushes 2023-10-02T12:32:38,541 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a. 2023-10-02T12:32:38,541 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a. 2023-10-02T12:32:38,541 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a. after waiting 0 ms 2023-10-02T12:32:38,541 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a. 2023-10-02T12:32:38,545 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns3/test-16962497916552/8f9314653c5767b76ceff134e893c29a/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-10-02T12:32:38,546 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:32:38,547 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a. 2023-10-02T12:32:38,547 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 8f9314653c5767b76ceff134e893c29a: 2023-10-02T12:32:38,547 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-16962497916552,,1696249910437.8f9314653c5767b76ceff134e893c29a. 2023-10-02T12:32:38,549 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing d30d346a1595b3fd84f804f2773895cc, disabling compactions & flushes 2023-10-02T12:32:38,549 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc. 2023-10-02T12:32:38,549 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc. 2023-10-02T12:32:38,549 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc. after waiting 0 ms 2023-10-02T12:32:38,549 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc. 2023-10-02T12:32:38,549 DEBUG [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 1ff8cee0e4d9c23004d17b9d828c2c96, 35dcfa6f2558560536edbd70539305b1, d30d346a1595b3fd84f804f2773895cc 2023-10-02T12:32:38,549 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing d30d346a1595b3fd84f804f2773895cc 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-10-02T12:32:38,566 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/namespace/d30d346a1595b3fd84f804f2773895cc/.tmp/info/ba96399290524766b022610c213b1f27 is 45, key is default/info:d/1696249786342/Put/seqid=0 2023-10-02T12:32:38,750 DEBUG [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 1ff8cee0e4d9c23004d17b9d828c2c96, 35dcfa6f2558560536edbd70539305b1, d30d346a1595b3fd84f804f2773895cc 2023-10-02T12:32:38,798 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.30 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740/.tmp/info/d64136d3833e462daef3554757f9b4b2 2023-10-02T12:32:38,826 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740/.tmp/rep_barrier/8cb119b51875450aad5d90a33135f037 is 107, key is ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9./rep_barrier:/1696249813428/DeleteFamily/seqid=0 2023-10-02T12:32:38,832 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.13 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740/.tmp/rep_barrier/8cb119b51875450aad5d90a33135f037 2023-10-02T12:32:38,854 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740/.tmp/table/5cf00be24ee546419190c5849ea7613b is 101, key is ns2:test-16962497916551,,1696249794418.58e28812c4354ed58ad888f86b1669f9./table:/1696249813428/DeleteFamily/seqid=0 2023-10-02T12:32:38,950 DEBUG [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(1609): Waiting on 1588230740, 1ff8cee0e4d9c23004d17b9d828c2c96, 35dcfa6f2558560536edbd70539305b1, d30d346a1595b3fd84f804f2773895cc 2023-10-02T12:32:38,973 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/namespace/d30d346a1595b3fd84f804f2773895cc/.tmp/info/ba96399290524766b022610c213b1f27 2023-10-02T12:32:38,979 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/namespace/d30d346a1595b3fd84f804f2773895cc/.tmp/info/ba96399290524766b022610c213b1f27 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/namespace/d30d346a1595b3fd84f804f2773895cc/info/ba96399290524766b022610c213b1f27 2023-10-02T12:32:38,984 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/namespace/d30d346a1595b3fd84f804f2773895cc/info/ba96399290524766b022610c213b1f27, entries=7, sequenceid=11, filesize=5.1 K 2023-10-02T12:32:38,985 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for d30d346a1595b3fd84f804f2773895cc in 436ms, sequenceid=11, compaction requested=false 2023-10-02T12:32:38,991 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/namespace/d30d346a1595b3fd84f804f2773895cc/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-10-02T12:32:38,992 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:32:38,993 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc. 2023-10-02T12:32:38,993 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for d30d346a1595b3fd84f804f2773895cc: 2023-10-02T12:32:38,993 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1696249785359.d30d346a1595b3fd84f804f2773895cc. 2023-10-02T12:32:38,993 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 1ff8cee0e4d9c23004d17b9d828c2c96, disabling compactions & flushes 2023-10-02T12:32:38,993 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96. 2023-10-02T12:32:38,993 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96. 2023-10-02T12:32:38,993 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96. after waiting 0 ms 2023-10-02T12:32:38,993 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96. 2023-10-02T12:32:38,999 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/ns2/test-16962497916551/1ff8cee0e4d9c23004d17b9d828c2c96/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2023-10-02T12:32:39,000 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:32:39,001 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96. 2023-10-02T12:32:39,001 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 1ff8cee0e4d9c23004d17b9d828c2c96: 2023-10-02T12:32:39,001 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-16962497916551,,1696249905552.1ff8cee0e4d9c23004d17b9d828c2c96. 2023-10-02T12:32:39,001 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 35dcfa6f2558560536edbd70539305b1, disabling compactions & flushes 2023-10-02T12:32:39,001 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1. 2023-10-02T12:32:39,002 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1. 2023-10-02T12:32:39,002 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1. after waiting 0 ms 2023-10-02T12:32:39,002 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1. 2023-10-02T12:32:39,005 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/default/test-1696249791655/35dcfa6f2558560536edbd70539305b1/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2023-10-02T12:32:39,005 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:32:39,006 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1. 2023-10-02T12:32:39,006 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 35dcfa6f2558560536edbd70539305b1: 2023-10-02T12:32:39,007 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1696249791655,,1696249860277.35dcfa6f2558560536edbd70539305b1. 2023-10-02T12:32:39,150 DEBUG [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(1609): Waiting on 1588230740 2023-10-02T12:32:39,177 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-10-02T12:32:39,177 INFO [regionserver/jenkins-hbase3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-10-02T12:32:39,261 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.56 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740/.tmp/table/5cf00be24ee546419190c5849ea7613b 2023-10-02T12:32:39,269 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740/.tmp/info/d64136d3833e462daef3554757f9b4b2 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740/info/d64136d3833e462daef3554757f9b4b2 2023-10-02T12:32:39,275 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740/info/d64136d3833e462daef3554757f9b4b2, entries=94, sequenceid=183, filesize=16.4 K 2023-10-02T12:32:39,277 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740/.tmp/rep_barrier/8cb119b51875450aad5d90a33135f037 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740/rep_barrier/8cb119b51875450aad5d90a33135f037 2023-10-02T12:32:39,287 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740/rep_barrier/8cb119b51875450aad5d90a33135f037, entries=12, sequenceid=183, filesize=6.5 K 2023-10-02T12:32:39,288 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740/.tmp/table/5cf00be24ee546419190c5849ea7613b as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740/table/5cf00be24ee546419190c5849ea7613b 2023-10-02T12:32:39,295 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740/table/5cf00be24ee546419190c5849ea7613b, entries=32, sequenceid=183, filesize=7.3 K 2023-10-02T12:32:39,297 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~38.00 KB/38910, heapSize ~66.25 KB/67840, currentSize=0 B/0 for 1588230740 in 949ms, sequenceid=183, compaction requested=false 2023-10-02T12:32:39,317 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/data/hbase/meta/1588230740/recovered.edits/186.seqid, newMaxSeqId=186, maxSeqId=1 2023-10-02T12:32:39,318 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:32:39,318 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-02T12:32:39,319 INFO [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-10-02T12:32:39,319 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-10-02T12:32:39,319 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-10-02T12:32:39,350 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,38255,1696249782114; all regions closed. 2023-10-02T12:32:39,355 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/WALs/jenkins-hbase3.apache.org,38255,1696249782114/jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta.1696249949459.meta not finished, retry = 0 2023-10-02T12:32:39,465 DEBUG [RS:0;jenkins-hbase3:38255 {}] wal.AbstractFSWAL(1062): Moved 10 WAL file(s) to /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/oldWALs 2023-10-02T12:32:39,465 INFO [RS:0;jenkins-hbase3:38255 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114.meta:.meta(num 1696249949459) 2023-10-02T12:32:39,474 DEBUG [RS:0;jenkins-hbase3:38255 {}] wal.AbstractFSWAL(1062): Moved 7 WAL file(s) to /user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/oldWALs 2023-10-02T12:32:39,474 INFO [RS:0;jenkins-hbase3:38255 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase3.apache.org%2C38255%2C1696249782114:(num 1696249949442) 2023-10-02T12:32:39,474 DEBUG [RS:0;jenkins-hbase3:38255 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:39,474 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.LeaseManager(133): Closed leases 2023-10-02T12:32:39,475 INFO [RS:0;jenkins-hbase3:38255 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-10-02T12:32:39,475 INFO [regionserver/jenkins-hbase3:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-10-02T12:32:39,477 INFO [RS:0;jenkins-hbase3:38255 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:38255 2023-10-02T12:32:39,487 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase3.apache.org,38255,1696249782114] 2023-10-02T12:32:39,488 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase3.apache.org,38255,1696249782114; numProcessing=1 2023-10-02T12:32:39,489 INFO [RegionServerTracker-0 {}] master.ServerManager(569): Cluster shutdown set; jenkins-hbase3.apache.org,38255,1696249782114 expired; onlineServers=0 2023-10-02T12:32:39,489 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase3.apache.org,40619,1696249778965' ***** 2023-10-02T12:32:39,489 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-10-02T12:32:39,490 DEBUG [M:0;jenkins-hbase3:40619 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@91ac29d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase3.apache.org/172.31.12.81:0 2023-10-02T12:32:39,490 INFO [M:0;jenkins-hbase3:40619 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-10-02T12:32:39,537 INFO [M:0;jenkins-hbase3:40619 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@5762bd3d{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-10-02T12:32:39,538 INFO [M:0;jenkins-hbase3:40619 {}] server.AbstractConnector(383): Stopped ServerConnector@35502a4a{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:32:39,538 INFO [M:0;jenkins-hbase3:40619 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-10-02T12:32:39,540 INFO [M:0;jenkins-hbase3:40619 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5dbdc559{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-10-02T12:32:39,541 INFO [M:0;jenkins-hbase3:40619 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@2cb278af{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/hadoop.log.dir/,STOPPED} 2023-10-02T12:32:39,543 INFO [M:0;jenkins-hbase3:40619 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase3.apache.org,40619,1696249778965 2023-10-02T12:32:39,543 INFO [M:0;jenkins-hbase3:40619 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase3.apache.org,40619,1696249778965; all regions closed. 2023-10-02T12:32:39,543 DEBUG [M:0;jenkins-hbase3:40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:39,543 INFO [M:0;jenkins-hbase3:40619 {}] master.HMaster(1665): Stopping master jetty server 2023-10-02T12:32:39,546 INFO [M:0;jenkins-hbase3:40619 {}] server.AbstractConnector(383): Stopped ServerConnector@eab1e7b{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-10-02T12:32:39,547 DEBUG [M:0;jenkins-hbase3:40619 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-10-02T12:32:39,547 INFO [M:0;jenkins-hbase3:40619 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-10-02T12:32:39,547 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-10-02T12:32:39,547 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696249783996 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.large.0-1696249783996,5,FailOnTimeoutGroup] 2023-10-02T12:32:39,547 DEBUG [M:0;jenkins-hbase3:40619 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-10-02T12:32:39,547 DEBUG [master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696249783997 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase3:0:becomeActiveMaster-HFileCleaner.small.0-1696249783997,5,FailOnTimeoutGroup] 2023-10-02T12:32:39,547 DEBUG [M:0;jenkins-hbase3:40619 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-10-02T12:32:39,548 INFO [M:0;jenkins-hbase3:40619 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase3:0 had [] on shutdown 2023-10-02T12:32:39,548 DEBUG [M:0;jenkins-hbase3:40619 {}] master.HMaster(1688): Stopping service threads 2023-10-02T12:32:39,548 INFO [M:0;jenkins-hbase3:40619 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-10-02T12:32:39,549 ERROR [M:0;jenkins-hbase3:40619 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-12,5,PEWorkerGroup] 2023-10-02T12:32:39,549 INFO [M:0;jenkins-hbase3:40619 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-10-02T12:32:39,550 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-10-02T12:32:39,552 WARN [M:0;jenkins-hbase3:40619 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-10-02T12:32:39,552 INFO [M:0;jenkins-hbase3:40619 {}] assignment.AssignmentManager(382): Stopping assignment manager 2023-10-02T12:32:39,553 INFO [M:0;jenkins-hbase3:40619 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-10-02T12:32:39,553 DEBUG [M:0;jenkins-hbase3:40619 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-10-02T12:32:39,569 INFO [M:0;jenkins-hbase3:40619 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:32:39,570 DEBUG [M:0;jenkins-hbase3:40619 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:32:39,570 DEBUG [M:0;jenkins-hbase3:40619 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-10-02T12:32:39,570 DEBUG [M:0;jenkins-hbase3:40619 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:32:39,570 INFO [M:0;jenkins-hbase3:40619 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=621.15 KB heapSize=755.27 KB 2023-10-02T12:32:39,587 DEBUG [M:0;jenkins-hbase3:40619 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b860da0416a341939ff91d26b52dcde3 is 88, key is hbase:meta,,1/info:sn/1696249784624/Put/seqid=0 2023-10-02T12:32:39,588 INFO [RS:0;jenkins-hbase3:38255 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,38255,1696249782114; zookeeper connection closed. 2023-10-02T12:32:39,589 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6c1d5648 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6c1d5648 2023-10-02T12:32:39,589 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-10-02T12:32:39,600 INFO [M:0;jenkins-hbase3:40619 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=1607 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b860da0416a341939ff91d26b52dcde3 2023-10-02T12:32:39,621 DEBUG [M:0;jenkins-hbase3:40619 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5b936468be594a7897f05781d0d8b7b1 is 987, key is \x00\x00\x00\x00\x00\x00\x00\x1D/proc:d/1696249800129/Put/seqid=0 2023-10-02T12:32:39,636 INFO [M:0;jenkins-hbase3:40619 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=620.56 KB at sequenceid=1607 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5b936468be594a7897f05781d0d8b7b1 2023-10-02T12:32:39,696 INFO [M:0;jenkins-hbase3:40619 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5b936468be594a7897f05781d0d8b7b1 2023-10-02T12:32:39,716 DEBUG [M:0;jenkins-hbase3:40619 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bf6dc99cdacb427f941df1e600ec5e25 is 82, key is jenkins-hbase3.apache.org,38255,1696249782114/rs:state/1696249784031/Put/seqid=0 2023-10-02T12:32:39,721 INFO [M:0;jenkins-hbase3:40619 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=1607 (bloomFilter=true), to=hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bf6dc99cdacb427f941df1e600ec5e25 2023-10-02T12:32:39,726 DEBUG [M:0;jenkins-hbase3:40619 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b860da0416a341939ff91d26b52dcde3 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b860da0416a341939ff91d26b52dcde3 2023-10-02T12:32:39,730 INFO [M:0;jenkins-hbase3:40619 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b860da0416a341939ff91d26b52dcde3, entries=8, sequenceid=1607, filesize=5.5 K 2023-10-02T12:32:39,730 DEBUG [M:0;jenkins-hbase3:40619 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5b936468be594a7897f05781d0d8b7b1 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5b936468be594a7897f05781d0d8b7b1 2023-10-02T12:32:39,734 INFO [M:0;jenkins-hbase3:40619 {}] regionserver.StoreFileReader(536): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5b936468be594a7897f05781d0d8b7b1 2023-10-02T12:32:39,734 INFO [M:0;jenkins-hbase3:40619 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5b936468be594a7897f05781d0d8b7b1, entries=173, sequenceid=1607, filesize=39.1 K 2023-10-02T12:32:39,735 DEBUG [M:0;jenkins-hbase3:40619 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bf6dc99cdacb427f941df1e600ec5e25 as hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bf6dc99cdacb427f941df1e600ec5e25 2023-10-02T12:32:39,740 INFO [M:0;jenkins-hbase3:40619 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:37823/user/jenkins/test-data/8c437d67-f8f4-2682-8ef1-3a9937cfde07/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bf6dc99cdacb427f941df1e600ec5e25, entries=1, sequenceid=1607, filesize=5.1 K 2023-10-02T12:32:39,741 INFO [M:0;jenkins-hbase3:40619 {}] regionserver.HRegion(3022): Finished flush of dataSize ~621.15 KB/636061, heapSize ~754.98 KB/773096, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 171ms, sequenceid=1607, compaction requested=false 2023-10-02T12:32:39,749 INFO [M:0;jenkins-hbase3:40619 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-10-02T12:32:39,749 DEBUG [M:0;jenkins-hbase3:40619 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-10-02T12:32:39,752 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-10-02T12:32:39,752 INFO [M:0;jenkins-hbase3:40619 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-10-02T12:32:39,752 INFO [M:0;jenkins-hbase3:40619 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-10-02T12:32:39,752 INFO [M:0;jenkins-hbase3:40619 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.12.81:40619 2023-10-02T12:32:39,856 INFO [M:0;jenkins-hbase3:40619 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase3.apache.org,40619,1696249778965; zookeeper connection closed. 2023-10-02T12:32:39,865 WARN [Listener at jenkins-hbase3.apache.org/38049 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-10-02T12:32:39,872 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-10-02T12:32:39,983 WARN [BP-2025135827-172.31.12.81-1696249775150 heartbeating to localhost/127.0.0.1:37823 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-10-02T12:32:39,983 WARN [BP-2025135827-172.31.12.81-1696249775150 heartbeating to localhost/127.0.0.1:37823 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-2025135827-172.31.12.81-1696249775150 (Datanode Uuid e74fb602-d5df-4cc6-94ee-2a855013b8e8) service to localhost/127.0.0.1:37823 2023-10-02T12:32:39,985 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/cluster_f5efbd98-923a-ea34-67ad-be7e4cbb4c4b/dfs/data/data1/current/BP-2025135827-172.31.12.81-1696249775150 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-02T12:32:39,985 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/9b3eec08-ec63-1797-806d-a35ea9449fc7/cluster_f5efbd98-923a-ea34-67ad-be7e4cbb4c4b/dfs/data/data2/current/BP-2025135827-172.31.12.81-1696249775150 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-10-02T12:32:40,018 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-10-02T12:32:40,134 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-10-02T12:32:40,134 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-10-02T12:32:40,135 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-16962497916553 2023-10-02T12:32:40,135 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-10-02T12:32:40,137 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-10-02T12:32:40,137 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-10-02T12:32:40,138 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-16962497916551 2023-10-02T12:32:40,138 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-10-02T12:32:40,139 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-16962497916552 2023-10-02T12:32:40,139 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-10-02T12:32:40,139 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1696249791655 2023-10-02T12:32:40,282 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-10-02T12:32:40,282 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-10-02T12:32:40,299 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:32:40,323 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:32:40,436 ERROR [Thread[Thread-275,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:32:40,436 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:32:40,543 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-10-02T12:32:40,554 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-10-02T12:32:40,556 ERROR [Thread[Thread-286,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:32:40,562 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase3.apache.org:0 2023-10-02T12:32:40,664 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-10-02T12:32:40,664 INFO [Listener at jenkins-hbase3.apache.org/38049 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped