2023-11-19 01:26:08,422 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3eb25e1a 2023-11-19 01:26:08,436 main DEBUG Took 0.011559 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2023-11-19 01:26:08,436 main DEBUG PluginManager 'Core' found 129 plugins 2023-11-19 01:26:08,437 main DEBUG PluginManager 'Level' found 0 plugins 2023-11-19 01:26:08,438 main DEBUG PluginManager 'Lookup' found 16 plugins 2023-11-19 01:26:08,439 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:26:08,447 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2023-11-19 01:26:08,460 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:26:08,462 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:26:08,463 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:26:08,463 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:26:08,464 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:26:08,464 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:26:08,465 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:26:08,465 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:26:08,466 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:26:08,466 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:26:08,467 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:26:08,468 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:26:08,468 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:26:08,469 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:26:08,469 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:26:08,470 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:26:08,470 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:26:08,471 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:26:08,471 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:26:08,472 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:26:08,472 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:26:08,473 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:26:08,474 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:26:08,474 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2023-11-19 01:26:08,474 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:26:08,475 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2023-11-19 01:26:08,478 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2023-11-19 01:26:08,480 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2023-11-19 01:26:08,483 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2023-11-19 01:26:08,483 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2023-11-19 01:26:08,485 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2023-11-19 01:26:08,485 main DEBUG PluginManager 'Converter' found 47 plugins 2023-11-19 01:26:08,503 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2023-11-19 01:26:08,506 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2023-11-19 01:26:08,511 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2023-11-19 01:26:08,511 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2023-11-19 01:26:08,512 main DEBUG createAppenders(={Console}) 2023-11-19 01:26:08,512 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3eb25e1a initialized 2023-11-19 01:26:08,513 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3eb25e1a 2023-11-19 01:26:08,513 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@3eb25e1a OK. 2023-11-19 01:26:08,513 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2023-11-19 01:26:08,514 main DEBUG OutputStream closed 2023-11-19 01:26:08,514 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2023-11-19 01:26:08,514 main DEBUG Appender DefaultConsole-1 stopped with status true 2023-11-19 01:26:08,514 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@59fd97a8 OK 2023-11-19 01:26:08,560 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586 2023-11-19 01:26:08,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=StatusLogger 2023-11-19 01:26:08,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=ContextSelector 2023-11-19 01:26:08,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name= 2023-11-19 01:26:08,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.directory 2023-11-19 01:26:08,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2023-11-19 01:26:08,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.zookeeper 2023-11-19 01:26:08,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2023-11-19 01:26:08,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2023-11-19 01:26:08,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2023-11-19 01:26:08,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase 2023-11-19 01:26:08,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop 2023-11-19 01:26:08,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2023-11-19 01:26:08,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2023-11-19 01:26:08,571 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2023-11-19 01:26:08,571 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2023-11-19 01:26:08,571 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2023-11-19 01:26:08,573 main DEBUG Registering MBean org.apache.logging.log4j2:type=1b6d3586,component=Appenders,name=Console 2023-11-19 01:26:08,576 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-11-19 01:26:08,576 main DEBUG Reconfiguration complete for context[name=1b6d3586] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.6.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@5403f35f) with optional ClassLoader: null 2023-11-19 01:26:08,577 main DEBUG Shutdown hook enabled. Registering a new one. 2023-11-19 01:26:08,578 main DEBUG LoggerContext[name=1b6d3586, org.apache.logging.log4j.core.LoggerContext@5403f35f] started OK. 2023-11-19T01:26:08,610 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.backup.TestFullRestore timeout: 13 mins 2023-11-19 01:26:08,615 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2023-11-19 01:26:08,615 main DEBUG org.apache.logging.log4j.core.util.SystemClock does not support precise timestamps. 2023-11-19T01:26:08,918 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554 2023-11-19T01:26:08,923 DEBUG [Time-limited test {}] impl.BackupManager(127): Added log cleaner: org.apache.hadoop.hbase.backup.master.BackupLogCleaner. Added master procedure manager: org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager.Added master procedure manager: org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-11-19T01:26:08,925 DEBUG [Time-limited test {}] impl.BackupManager(157): Added region procedure manager: org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager. Added region observer: org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:26:08,926 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2023-11-19T01:26:08,930 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/cluster_6f56ccd7-1fd7-2f9a-c808-0f86c08a2321, deleteOnExit=true 2023-11-19T01:26:08,930 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2023-11-19T01:26:08,931 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/test.cache.data in system properties and HBase conf 2023-11-19T01:26:08,931 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop.tmp.dir in system properties and HBase conf 2023-11-19T01:26:08,931 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop.log.dir in system properties and HBase conf 2023-11-19T01:26:08,932 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-19T01:26:08,932 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-19T01:26:08,932 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-19T01:26:09,046 WARN [Time-limited test {}] util.NativeCodeLoader(62): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2023-11-19T01:26:09,559 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2023-11-19T01:26:09,564 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:26:09,564 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:26:09,564 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-19T01:26:09,565 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:26:09,565 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-19T01:26:09,566 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-19T01:26:09,566 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:26:09,566 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:26:09,567 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-19T01:26:09,567 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/nfs.dump.dir in system properties and HBase conf 2023-11-19T01:26:09,567 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/java.io.tmpdir in system properties and HBase conf 2023-11-19T01:26:09,567 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:26:09,568 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-19T01:26:09,568 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/fs.s3a.committer.staging.tmp.path in system properties and HBase conf Formatting using clusterid: testClusterID 2023-11-19T01:26:10,038 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-19T01:26:10,043 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-19T01:26:10,335 WARN [Time-limited test {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2023-11-19T01:26:10,502 INFO [Time-limited test {}] log.Slf4jLog(67): Logging to org.apache.logging.slf4j.Log4jLogger@3abc60ad via org.mortbay.log.Slf4jLog 2023-11-19T01:26:10,525 WARN [Time-limited test {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:26:10,572 INFO [Time-limited test {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:26:10,610 INFO [Time-limited test {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/hdfs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/java.io.tmpdir/Jetty_localhost_35231_hdfs____w16v4o/webapp 2023-11-19T01:26:10,767 INFO [Time-limited test {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:35231 2023-11-19T01:26:10,789 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.heartbeat.interval(3) assuming SECONDS 2023-11-19T01:26:10,789 WARN [Time-limited test {}] conf.Configuration(1733): No unit for dfs.namenode.safemode.extension(0) assuming MILLISECONDS 2023-11-19T01:26:11,376 WARN [Listener at localhost/42739 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-19T01:26:11,445 WARN [Listener at localhost/42739 {}] conf.Configuration(1733): No unit for dfs.datanode.outliers.report.interval(1800000) assuming MILLISECONDS 2023-11-19T01:26:11,465 WARN [Listener at localhost/42739 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:26:11,471 INFO [Listener at localhost/42739 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:26:11,478 INFO [Listener at localhost/42739 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/2.10.2/hadoop-hdfs-2.10.2-tests.jar!/webapps/datanode to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/java.io.tmpdir/Jetty_localhost_40887_datanode____.ubi4wu/webapp 2023-11-19T01:26:11,583 INFO [Listener at localhost/42739 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:40887 2023-11-19T01:26:11,868 WARN [Listener at localhost/35765 {}] common.MetricsLoggerTask(153): Metrics logging will not be async since the logger is not log4j 2023-11-19T01:26:12,181 WARN [Thread-52 {}] conf.Configuration(1733): No unit for dfs.datanode.lock-reporting-threshold-ms(300) assuming MILLISECONDS 2023-11-19T01:26:12,377 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x10da6059d9b96bd4: Processing first storage report for DS-0e894dbf-f050-45c4-aafb-6175da034d96 from datanode 689d6bea-b58f-433c-b8d6-456502b71baf 2023-11-19T01:26:12,379 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x10da6059d9b96bd4: from storage DS-0e894dbf-f050-45c4-aafb-6175da034d96 node DatanodeRegistration(127.0.0.1:43079, datanodeUuid=689d6bea-b58f-433c-b8d6-456502b71baf, infoPort=38791, infoSecurePort=0, ipcPort=35765, storageInfo=lv=-57;cid=testClusterID;nsid=1507914448;c=1700357170119), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2023-11-19T01:26:12,379 INFO [Block report processor {}] blockmanagement.BlockManager(2202): BLOCK* processReport 0x10da6059d9b96bd4: Processing first storage report for DS-c34860b6-9e22-41eb-b34b-6512308116d8 from datanode 689d6bea-b58f-433c-b8d6-456502b71baf 2023-11-19T01:26:12,380 INFO [Block report processor {}] blockmanagement.BlockManager(2228): BLOCK* processReport 0x10da6059d9b96bd4: from storage DS-c34860b6-9e22-41eb-b34b-6512308116d8 node DatanodeRegistration(127.0.0.1:43079, datanodeUuid=689d6bea-b58f-433c-b8d6-456502b71baf, infoPort=38791, infoSecurePort=0, ipcPort=35765, storageInfo=lv=-57;cid=testClusterID;nsid=1507914448;c=1700357170119), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2023-11-19T01:26:12,469 DEBUG [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554 2023-11-19T01:26:12,614 INFO [Listener at localhost/35765 {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/cluster_6f56ccd7-1fd7-2f9a-c808-0f86c08a2321/zookeeper_0, clientPort=57501, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/cluster_6f56ccd7-1fd7-2f9a-c808-0f86c08a2321/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/cluster_6f56ccd7-1fd7-2f9a-c808-0f86c08a2321/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2023-11-19T01:26:12,635 INFO [Listener at localhost/35765 {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=57501 2023-11-19T01:26:12,643 INFO [Listener at localhost/35765 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:26:12,645 INFO [Listener at localhost/35765 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:26:13,324 INFO [Listener at localhost/35765 {}] util.FSUtils(462): Created version file at hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af with version=8 2023-11-19T01:26:13,325 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging 2023-11-19T01:26:13,347 DEBUG [Listener at localhost/35765 {}] hbase.LocalHBaseCluster(134): Setting Master Port to random. 2023-11-19T01:26:13,347 DEBUG [Listener at localhost/35765 {}] hbase.LocalHBaseCluster(141): Setting RegionServer Port to random. 2023-11-19T01:26:13,347 DEBUG [Listener at localhost/35765 {}] hbase.LocalHBaseCluster(151): Setting RS InfoServer Port to random. 2023-11-19T01:26:13,348 DEBUG [Listener at localhost/35765 {}] hbase.LocalHBaseCluster(159): Setting Master InfoServer Port to random. 2023-11-19T01:26:13,588 DEBUG [Listener at localhost/35765 {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2023-11-19T01:26:14,429 INFO [Listener at localhost/35765 {}] client.ConnectionUtils(128): master/jenkins-hbase6:0 server-side Connection retries=45 2023-11-19T01:26:14,480 INFO [Listener at localhost/35765 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-19T01:26:14,481 INFO [Listener at localhost/35765 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-19T01:26:14,481 INFO [Listener at localhost/35765 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-19T01:26:14,481 INFO [Listener at localhost/35765 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-19T01:26:14,481 INFO [Listener at localhost/35765 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-19T01:26:14,636 INFO [Listener at localhost/35765 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-19T01:26:14,791 INFO [Listener at localhost/35765 {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2023-11-19T01:26:14,819 DEBUG [Listener at localhost/35765 {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2023-11-19T01:26:14,826 INFO [Listener at localhost/35765 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-19T01:26:14,877 DEBUG [Listener at localhost/35765 {}] channel.DefaultChannelId(79): -Dio.netty.processId: 3002 (auto-detected) 2023-11-19T01:26:14,878 DEBUG [Listener at localhost/35765 {}] channel.DefaultChannelId(101): -Dio.netty.machineId: 02:42:55:ff:fe:37:d3:4d (auto-detected) 2023-11-19T01:26:14,915 INFO [Listener at localhost/35765 {}] ipc.NettyRpcServer(192): Bind to /172.31.2.20:44889 2023-11-19T01:26:14,929 INFO [Listener at localhost/35765 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:26:14,931 INFO [Listener at localhost/35765 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:26:14,953 INFO [Listener at localhost/35765 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=master:44889 connecting to ZooKeeper ensemble=127.0.0.1:57501 2023-11-19T01:26:15,006 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:448890x0, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-19T01:26:15,009 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): master:44889-0x10123a379ab0000 connected 2023-11-19T01:26:15,045 DEBUG [Listener at localhost/35765 {}] zookeeper.ZKUtil(113): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-19T01:26:15,049 DEBUG [Listener at localhost/35765 {}] zookeeper.ZKUtil(113): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-19T01:26:15,052 DEBUG [Listener at localhost/35765 {}] zookeeper.ZKUtil(113): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-11-19T01:26:15,066 DEBUG [Listener at localhost/35765 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44889 2023-11-19T01:26:15,067 DEBUG [Listener at localhost/35765 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44889 2023-11-19T01:26:15,067 DEBUG [Listener at localhost/35765 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44889 2023-11-19T01:26:15,068 DEBUG [Listener at localhost/35765 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44889 2023-11-19T01:26:15,068 DEBUG [Listener at localhost/35765 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44889 2023-11-19T01:26:15,124 INFO [Listener at localhost/35765 {}] log.Log(170): Logging initialized @7520ms to org.apache.hbase.thirdparty.org.eclipse.jetty.util.log.Slf4jLog 2023-11-19T01:26:15,262 INFO [Listener at localhost/35765 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-19T01:26:15,262 INFO [Listener at localhost/35765 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-19T01:26:15,263 INFO [Listener at localhost/35765 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-19T01:26:15,265 INFO [Listener at localhost/35765 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context master 2023-11-19T01:26:15,265 INFO [Listener at localhost/35765 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-19T01:26:15,266 INFO [Listener at localhost/35765 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-19T01:26:15,269 INFO [Listener at localhost/35765 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-19T01:26:15,413 INFO [Listener at localhost/35765 {}] http.HttpServer(1227): Jetty bound to port 45975 2023-11-19T01:26:15,415 INFO [Listener at localhost/35765 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-19T01:26:15,447 INFO [Listener at localhost/35765 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:26:15,451 INFO [Listener at localhost/35765 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@2b034ac3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop.log.dir/,AVAILABLE} 2023-11-19T01:26:15,451 INFO [Listener at localhost/35765 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:26:15,452 INFO [Listener at localhost/35765 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@f63ec6a{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-19T01:26:15,516 INFO [Listener at localhost/35765 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-19T01:26:15,530 INFO [Listener at localhost/35765 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-19T01:26:15,531 INFO [Listener at localhost/35765 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-19T01:26:15,533 INFO [Listener at localhost/35765 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-11-19T01:26:15,540 INFO [Listener at localhost/35765 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:26:16,702 INFO [Listener at localhost/35765 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@38b9796f{master,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-19T01:26:16,715 INFO [Listener at localhost/35765 {}] server.AbstractConnector(333): Started ServerConnector@7750fcaa{HTTP/1.1, (http/1.1)}{0.0.0.0:45975} 2023-11-19T01:26:16,716 INFO [Listener at localhost/35765 {}] server.Server(415): Started @9111ms 2023-11-19T01:26:16,720 INFO [Listener at localhost/35765 {}] master.HMaster(486): hbase.rootdir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af, hbase.cluster.distributed=false 2023-11-19T01:26:16,785 INFO [Listener at localhost/35765 {}] client.ConnectionUtils(128): regionserver/jenkins-hbase6:0 server-side Connection retries=45 2023-11-19T01:26:16,785 INFO [Listener at localhost/35765 {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-19T01:26:16,785 INFO [Listener at localhost/35765 {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2023-11-19T01:26:16,785 INFO [Listener at localhost/35765 {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2023-11-19T01:26:16,786 INFO [Listener at localhost/35765 {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2023-11-19T01:26:16,786 INFO [Listener at localhost/35765 {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2023-11-19T01:26:16,795 INFO [Listener at localhost/35765 {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService 2023-11-19T01:26:16,797 INFO [Listener at localhost/35765 {}] ipc.NettyRpcServer(310): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2023-11-19T01:26:16,798 INFO [Listener at localhost/35765 {}] ipc.NettyRpcServer(192): Bind to /172.31.2.20:44997 2023-11-19T01:26:16,801 INFO [Listener at localhost/35765 {}] hfile.BlockCacheFactory(127): Allocating BlockCache size=782.40 MB, blockSize=64 KB 2023-11-19T01:26:16,808 DEBUG [Listener at localhost/35765 {}] mob.MobFileCache(121): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2023-11-19T01:26:16,809 INFO [Listener at localhost/35765 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:26:16,812 INFO [Listener at localhost/35765 {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:26:16,814 INFO [Listener at localhost/35765 {}] zookeeper.RecoverableZooKeeper(142): Process identifier=regionserver:44997 connecting to ZooKeeper ensemble=127.0.0.1:57501 2023-11-19T01:26:16,818 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:449970x0, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2023-11-19T01:26:16,819 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(630): regionserver:44997-0x10123a379ab0001 connected 2023-11-19T01:26:16,819 DEBUG [Listener at localhost/35765 {}] zookeeper.ZKUtil(113): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/master 2023-11-19T01:26:16,821 DEBUG [Listener at localhost/35765 {}] zookeeper.ZKUtil(113): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/running 2023-11-19T01:26:16,821 DEBUG [Listener at localhost/35765 {}] zookeeper.ZKUtil(113): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/acl 2023-11-19T01:26:16,822 DEBUG [Listener at localhost/35765 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44997 2023-11-19T01:26:16,822 DEBUG [Listener at localhost/35765 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44997 2023-11-19T01:26:16,823 DEBUG [Listener at localhost/35765 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44997 2023-11-19T01:26:16,823 DEBUG [Listener at localhost/35765 {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44997 2023-11-19T01:26:16,823 DEBUG [Listener at localhost/35765 {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44997 2023-11-19T01:26:16,827 INFO [Listener at localhost/35765 {}] http.HttpServer(981): Added global filter 'safety' (class=org.apache.hadoop.hbase.http.HttpServer$QuotingInputFilter) 2023-11-19T01:26:16,827 INFO [Listener at localhost/35765 {}] http.HttpServer(981): Added global filter 'clickjackingprevention' (class=org.apache.hadoop.hbase.http.ClickjackingPreventionFilter) 2023-11-19T01:26:16,827 INFO [Listener at localhost/35765 {}] http.HttpServer(981): Added global filter 'securityheaders' (class=org.apache.hadoop.hbase.http.SecurityHeadersFilter) 2023-11-19T01:26:16,828 INFO [Listener at localhost/35765 {}] http.HttpServer(960): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context regionserver 2023-11-19T01:26:16,828 INFO [Listener at localhost/35765 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context logs 2023-11-19T01:26:16,828 INFO [Listener at localhost/35765 {}] http.HttpServer(967): Added filter static_user_filter (class=org.apache.hadoop.hbase.http.lib.StaticUserWebFilter$StaticUserFilter) to context static 2023-11-19T01:26:16,828 INFO [Listener at localhost/35765 {}] http.HttpServer(805): ASYNC_PROFILER_HOME environment variable and async.profiler.home system property not specified. Disabling /prof endpoint. 2023-11-19T01:26:16,831 INFO [Listener at localhost/35765 {}] http.HttpServer(1227): Jetty bound to port 45835 2023-11-19T01:26:16,831 INFO [Listener at localhost/35765 {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-19T01:26:16,835 INFO [Listener at localhost/35765 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:26:16,835 INFO [Listener at localhost/35765 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@50332b50{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop.log.dir/,AVAILABLE} 2023-11-19T01:26:16,836 INFO [Listener at localhost/35765 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:26:16,836 INFO [Listener at localhost/35765 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.s.ServletContextHandler@5ab6f613{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,AVAILABLE} 2023-11-19T01:26:16,849 INFO [Listener at localhost/35765 {}] webapp.StandardDescriptorProcessor(277): NO JSP Support for /, did not find org.apache.hbase.thirdparty.org.eclipse.jetty.jsp.JettyJspServlet 2023-11-19T01:26:16,850 INFO [Listener at localhost/35765 {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2023-11-19T01:26:16,850 INFO [Listener at localhost/35765 {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2023-11-19T01:26:16,851 INFO [Listener at localhost/35765 {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2023-11-19T01:26:16,852 INFO [Listener at localhost/35765 {}] http.SecurityHeadersFilter(48): Added security headers filter 2023-11-19T01:26:16,856 INFO [Listener at localhost/35765 {}] handler.ContextHandler(921): Started o.a.h.t.o.e.j.w.WebAppContext@52704e90{regionserver,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver/,AVAILABLE}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-19T01:26:16,858 INFO [Listener at localhost/35765 {}] server.AbstractConnector(333): Started ServerConnector@65b85596{HTTP/1.1, (http/1.1)}{0.0.0.0:45835} 2023-11-19T01:26:16,858 INFO [Listener at localhost/35765 {}] server.Server(415): Started @9254ms 2023-11-19T01:26:16,863 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] server.Server(375): jetty-9.4.52.v20230823; built: 2023-08-23T19:29:37.669Z; git: abdcda73818a1a2c705da276edb0bf6581e7997e; jvm 1.8.0_362-b09 2023-11-19T01:26:16,870 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] server.AbstractConnector(333): Started ServerConnector@4b97ab40{HTTP/1.1, (http/1.1)}{0.0.0.0:33987} 2023-11-19T01:26:16,870 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] server.Server(415): Started @9266ms 2023-11-19T01:26:16,870 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(2389): Adding backup master ZNode /1/backup-masters/jenkins-hbase6.apache.org,44889,1700357173572 2023-11-19T01:26:16,887 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-19T01:26:16,887 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-19T01:26:16,891 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on existing znode=/1/backup-masters/jenkins-hbase6.apache.org,44889,1700357173572 2023-11-19T01:26:16,893 DEBUG [M:0;jenkins-hbase6:44889 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;jenkins-hbase6:44889 2023-11-19T01:26:16,914 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-11-19T01:26:16,914 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/master 2023-11-19T01:26:16,914 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:26:16,914 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:26:16,915 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on existing znode=/1/master 2023-11-19T01:26:16,917 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /1/backup-masters/jenkins-hbase6.apache.org,44889,1700357173572 from backup master directory 2023-11-19T01:26:16,918 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on existing znode=/1/master 2023-11-19T01:26:16,921 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/backup-masters/jenkins-hbase6.apache.org,44889,1700357173572 2023-11-19T01:26:16,921 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-19T01:26:16,921 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/backup-masters 2023-11-19T01:26:16,923 WARN [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-19T01:26:16,923 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=jenkins-hbase6.apache.org,44889,1700357173572 2023-11-19T01:26:16,927 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 352, initial count 0 2023-11-19T01:26:16,929 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 391, initial count 0 2023-11-19T01:26:17,057 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] util.FSUtils(611): Created cluster ID file at hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase.id with ID: 1a37bdbe-883f-422e-b0eb-8a830834c896 2023-11-19T01:26:17,099 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2023-11-19T01:26:17,117 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:26:17,117 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:26:17,161 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegion(355): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:26:17,164 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2023-11-19T01:26:17,185 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(271): ClientProtocol::create wrong number of arguments, should be hadoop 3.2 or below 2023-11-19T01:26:17,185 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(277): ClientProtocol::create wrong number of arguments, should be hadoop 2.x 2023-11-19T01:26:17,186 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(286): can not find SHOULD_REPLICATE flag, should be hadoop 2.x java.lang.IllegalArgumentException: No enum constant org.apache.hadoop.fs.CreateFlag.SHOULD_REPLICATE at java.lang.Enum.valueOf(Enum.java:238) ~[?:1.8.0_362] at org.apache.hadoop.fs.CreateFlag.valueOf(CreateFlag.java:63) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.loadShouldReplicateFlag(FanOutOneBlockAsyncDFSOutputHelper.java:284) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.(FanOutOneBlockAsyncDFSOutputHelper.java:311) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:140) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:952) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2414) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:575) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:572) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-19T01:26:17,198 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2130) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:141) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:385) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:952) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2414) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$null$0(HMaster.java:575) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:572) ~[classes/:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-19T01:26:17,200 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-19T01:26:17,273 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7093): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/data/master/store 2023-11-19T01:26:17,298 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2023-11-19T01:26:17,298 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:17,299 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-19T01:26:17,299 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:26:17,299 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:26:17,299 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-19T01:26:17,299 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:26:17,300 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:26:17,300 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-19T01:26:17,302 WARN [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegion(232): failed to clean up initializing flag: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/data/master/store/.initializing 2023-11-19T01:26:17,302 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegion(202): WALDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/WALs/jenkins-hbase6.apache.org,44889,1700357173572 2023-11-19T01:26:17,314 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-11-19T01:26:17,330 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase6.apache.org%2C44889%2C1700357173572, suffix=, logDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/WALs/jenkins-hbase6.apache.org,44889,1700357173572, archiveDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/oldWALs, maxLogs=10 2023-11-19T01:26:17,364 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/WALs/jenkins-hbase6.apache.org,44889,1700357173572/jenkins-hbase6.apache.org%2C44889%2C1700357173572.1700357177337, exclude list is [], retry=0 2023-11-19T01:26:17,403 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:26:17,414 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(123): Hadoop 3.2 and below use unshaded protobuf. java.lang.ClassNotFoundException: org.apache.hadoop.thirdparty.protobuf.MessageLite at java.net.URLClassLoader.findClass(URLClassLoader.java:387) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:418) ~[?:1.8.0_362] at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:352) ~[?:1.8.0_362] at java.lang.ClassLoader.loadClass(ClassLoader.java:351) ~[?:1.8.0_362] at java.lang.Class.forName0(Native Method) ~[?:1.8.0_362] at java.lang.Class.forName(Class.java:264) ~[?:1.8.0_362] at org.apache.hadoop.hbase.io.asyncfs.ProtobufDecoder.(ProtobufDecoder.java:118) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.processWriteBlockResponse(FanOutOneBlockAsyncDFSOutputHelper.java:347) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$100(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$4.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:431) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:557) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:185) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.addListener(DefaultPromise.java:35) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hadoop.hbase.util.NettyFutureUtils.addListener(NettyFutureUtils.java:52) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:425) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.access$300(FanOutOneBlockAsyncDFSOutputHelper.java:121) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:483) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:478) ~[hbase-asyncfs-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:653) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:691) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:499) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:407) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.5.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.5.jar:?] at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362] 2023-11-19T01:26:17,511 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/WALs/jenkins-hbase6.apache.org,44889,1700357173572/jenkins-hbase6.apache.org%2C44889%2C1700357173572.1700357177337 2023-11-19T01:26:17,512 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:26:17,513 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7254): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:26:17,513 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(888): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:17,519 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7296): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:26:17,521 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(7299): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:26:17,599 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:26:17,638 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2023-11-19T01:26:17,644 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:17,648 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:26:17,649 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:26:17,652 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2023-11-19T01:26:17,652 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:17,653 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:26:17,653 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:26:17,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2023-11-19T01:26:17,656 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:17,657 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:26:17,657 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:26:17,659 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2023-11-19T01:26:17,660 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:17,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(324): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:26:17,665 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:26:17,667 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:26:17,684 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2023-11-19T01:26:17,689 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1076): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2023-11-19T01:26:17,693 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:26:17,694 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(1093): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10985927680, jitterRate=0.023144245147705078}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2023-11-19T01:26:17,700 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] regionserver.HRegion(992): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-19T01:26:17,701 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2023-11-19T01:26:17,743 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f05af06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:26:17,803 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(861): No meta location available on zookeeper, skip migrating... 2023-11-19T01:26:17,823 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2023-11-19T01:26:17,824 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(563): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2023-11-19T01:26:17,827 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2023-11-19T01:26:17,828 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(583): Recovered RegionProcedureStore lease in 1 msec 2023-11-19T01:26:17,834 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(597): Loaded RegionProcedureStore in 6 msec 2023-11-19T01:26:17,835 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2023-11-19T01:26:17,872 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2023-11-19T01:26:17,892 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Unable to get data of znode /1/balancer because node does not exist (not necessarily an error) 2023-11-19T01:26:17,894 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/balancer already deleted, retry=false 2023-11-19T01:26:17,897 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2023-11-19T01:26:17,899 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Unable to get data of znode /1/normalizer because node does not exist (not necessarily an error) 2023-11-19T01:26:17,901 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/normalizer already deleted, retry=false 2023-11-19T01:26:17,904 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2023-11-19T01:26:17,912 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Unable to get data of znode /1/switch/split because node does not exist (not necessarily an error) 2023-11-19T01:26:17,913 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/split already deleted, retry=false 2023-11-19T01:26:17,915 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Unable to get data of znode /1/switch/merge because node does not exist (not necessarily an error) 2023-11-19T01:26:17,916 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/switch/merge already deleted, retry=false 2023-11-19T01:26:17,933 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Unable to get data of znode /1/snapshot-cleanup because node does not exist (not necessarily an error) 2023-11-19T01:26:17,935 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(224): Node /1/snapshot-cleanup already deleted, retry=false 2023-11-19T01:26:17,941 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-11-19T01:26:17,941 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:26:17,943 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/running 2023-11-19T01:26:17,943 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:26:17,943 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(805): Active/primary master=jenkins-hbase6.apache.org,44889,1700357173572, sessionid=0x10123a379ab0000, setting cluster-up flag (Was=false) 2023-11-19T01:26:17,953 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager was loaded successfully. 2023-11-19T01:26:17,965 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:26:17,965 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:26:17,973 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/flush-table-proc/acquired, /1/flush-table-proc/reached, /1/flush-table-proc/abort 2023-11-19T01:26:17,975 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase6.apache.org,44889,1700357173572 2023-11-19T01:26:17,979 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:26:17,979 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:26:17,991 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/rolllog-proc/acquired, /1/rolllog-proc/reached, /1/rolllog-proc/abort 2023-11-19T01:26:17,993 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase6.apache.org,44889,1700357173572 2023-11-19T01:26:17,999 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:26:17,999 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:26:18,006 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /1/online-snapshot/acquired, /1/online-snapshot/reached, /1/online-snapshot/abort 2023-11-19T01:26:18,007 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=jenkins-hbase6.apache.org,44889,1700357173572 2023-11-19T01:26:18,081 DEBUG [RS:0;jenkins-hbase6:44997 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;jenkins-hbase6:44997 2023-11-19T01:26:18,083 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(997): ClusterId : 1a37bdbe-883f-422e-b0eb-8a830834c896 2023-11-19T01:26:18,085 INFO [RS:0;jenkins-hbase6:44997 {}] procedure.ProcedureManagerHost(66): User procedure org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager was loaded successfully. 2023-11-19T01:26:18,087 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2023-11-19T01:26:18,094 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2023-11-19T01:26:18,094 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.RegionServerProcedureManagerHost(43): Procedure backup-proc initializing 2023-11-19T01:26:18,097 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.RegionServerProcedureManagerHost(45): Procedure backup-proc initialized 2023-11-19T01:26:18,097 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2023-11-19T01:26:18,101 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2023-11-19T01:26:18,102 DEBUG [RS:0;jenkins-hbase6:44997 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45eebe02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:26:18,104 DEBUG [RS:0;jenkins-hbase6:44997 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ef7c985, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase6.apache.org/172.31.2.20:0 2023-11-19T01:26:18,112 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2023-11-19T01:26:18,112 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2023-11-19T01:26:18,112 DEBUG [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1079): About to register with Master. 2023-11-19T01:26:18,116 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase6.apache.org,44889,1700357173572 with isa=jenkins-hbase6.apache.org/172.31.2.20:44997, startcode=1700357176784 2023-11-19T01:26:18,144 DEBUG [RS:0;jenkins-hbase6:44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-19T01:26:18,205 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2023-11-19T01:26:18,218 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(574): slop=0.2 2023-11-19T01:26:18,226 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(285): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2023-11-19T01:26:18,230 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43831, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-11-19T01:26:18,244 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: jenkins-hbase6.apache.org,44889,1700357173572 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2023-11-19T01:26:18,248 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44889 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3212) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:592) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:17393) ~[hbase-protocol-shaded-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:437) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.6.0-SNAPSHOT] 2023-11-19T01:26:18,250 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2023-11-19T01:26:18,250 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2023-11-19T01:26:18,250 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2023-11-19T01:26:18,250 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=5, maxPoolSize=5 2023-11-19T01:26:18,250 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/jenkins-hbase6:0, corePoolSize=10, maxPoolSize=10 2023-11-19T01:26:18,251 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:26:18,251 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=2, maxPoolSize=2 2023-11-19T01:26:18,251 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:26:18,256 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1700357208256 2023-11-19T01:26:18,260 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2023-11-19T01:26:18,269 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2023-11-19T01:26:18,278 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2023-11-19T01:26:18,283 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2023-11-19T01:26:18,288 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2023-11-19T01:26:18,289 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.master.BackupLogCleaner 2023-11-19T01:26:18,290 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2023-11-19T01:26:18,290 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2023-11-19T01:26:18,291 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2023-11-19T01:26:18,291 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-19T01:26:18,293 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:18,294 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-19T01:26:18,301 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2023-11-19T01:26:18,305 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2023-11-19T01:26:18,306 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2023-11-19T01:26:18,308 DEBUG [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(3046): Master is not running yet 2023-11-19T01:26:18,309 WARN [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1088): reportForDuty failed; sleeping 100 ms and then retrying. 2023-11-19T01:26:18,331 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4a94647d to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:26:18,332 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2023-11-19T01:26:18,333 INFO [PEWorker-1 {}] regionserver.HRegion(7075): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:26:18,351 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a479318, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:26:18,351 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.backup.BackupHFileCleaner 2023-11-19T01:26:18,356 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2023-11-19T01:26:18,358 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2023-11-19T01:26:18,363 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.large.0-1700357178363,5,FailOnTimeoutGroup] 2023-11-19T01:26:18,363 DEBUG [PEWorker-1 {}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:18,366 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.small.0-1700357178363,5,FailOnTimeoutGroup] 2023-11-19T01:26:18,367 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2023-11-19T01:26:18,367 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1636): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2023-11-19T01:26:18,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-19T01:26:18,369 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2023-11-19T01:26:18,369 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2023-11-19T01:26:18,369 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-19T01:26:18,370 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:18,370 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:26:18,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-19T01:26:18,373 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-19T01:26:18,373 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:18,374 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:26:18,374 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-19T01:26:18,376 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-19T01:26:18,376 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:18,377 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:26:18,379 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740 2023-11-19T01:26:18,379 DEBUG [PEWorker-1 {}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740 2023-11-19T01:26:18,383 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-19T01:26:18,385 DEBUG [PEWorker-1 {}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-19T01:26:18,389 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:26:18,389 INFO [PEWorker-1 {}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9477751360, jitterRate=-0.1173156201839447}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-19T01:26:18,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-19T01:26:18,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-19T01:26:18,392 INFO [PEWorker-1 {}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-19T01:26:18,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-19T01:26:18,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-19T01:26:18,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-19T01:26:18,393 INFO [PEWorker-1 {}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-19T01:26:18,394 DEBUG [PEWorker-1 {}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-19T01:26:18,397 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2023-11-19T01:26:18,397 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2023-11-19T01:26:18,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2023-11-19T01:26:18,410 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(3027): reportForDuty to master=jenkins-hbase6.apache.org,44889,1700357173572 with isa=jenkins-hbase6.apache.org/172.31.2.20:44997, startcode=1700357176784 2023-11-19T01:26:18,415 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44889 {}] master.ServerManager(396): Registering regionserver=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:18,421 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2023-11-19T01:26:18,430 DEBUG [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1705): Config from master: hbase.rootdir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:26:18,430 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2023-11-19T01:26:18,430 DEBUG [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1705): Config from master: fs.defaultFS=hdfs://localhost:42739 2023-11-19T01:26:18,430 DEBUG [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1705): Config from master: hbase.master.info.port=45975 2023-11-19T01:26:18,446 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rs 2023-11-19T01:26:18,447 DEBUG [RS:0;jenkins-hbase6:44997 {}] zookeeper.ZKUtil(111): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on existing znode=/1/rs/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:18,447 WARN [RS:0;jenkins-hbase6:44997 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2023-11-19T01:26:18,447 INFO [RS:0;jenkins-hbase6:44997 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-19T01:26:18,447 DEBUG [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(2060): logDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:18,450 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [jenkins-hbase6.apache.org,44997,1700357176784] 2023-11-19T01:26:18,466 DEBUG [RS:0;jenkins-hbase6:44997 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2023-11-19T01:26:18,482 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.MetricsRegionServerWrapperImpl(165): Computing regionserver metrics every 5000 milliseconds 2023-11-19T01:26:18,496 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=782.4 M, globalMemStoreLimitLowMark=743.3 M, Offheap=false 2023-11-19T01:26:18,500 INFO [RS:0;jenkins-hbase6:44997 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2023-11-19T01:26:18,500 INFO [RS:0;jenkins-hbase6:44997 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2023-11-19T01:26:18,501 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer$CompactionChecker(1947): CompactionChecker runs every PT1S 2023-11-19T01:26:18,510 INFO [RS:0;jenkins-hbase6:44997 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2023-11-19T01:26:18,511 DEBUG [RS:0;jenkins-hbase6:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:26:18,511 DEBUG [RS:0;jenkins-hbase6:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:26:18,511 DEBUG [RS:0;jenkins-hbase6:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:26:18,511 DEBUG [RS:0;jenkins-hbase6:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:26:18,511 DEBUG [RS:0;jenkins-hbase6:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:26:18,511 DEBUG [RS:0;jenkins-hbase6:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/jenkins-hbase6:0, corePoolSize=2, maxPoolSize=2 2023-11-19T01:26:18,511 DEBUG [RS:0;jenkins-hbase6:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:26:18,512 DEBUG [RS:0;jenkins-hbase6:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:26:18,512 DEBUG [RS:0;jenkins-hbase6:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:26:18,512 DEBUG [RS:0;jenkins-hbase6:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:26:18,512 DEBUG [RS:0;jenkins-hbase6:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/jenkins-hbase6:0, corePoolSize=1, maxPoolSize=1 2023-11-19T01:26:18,512 DEBUG [RS:0;jenkins-hbase6:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0, corePoolSize=3, maxPoolSize=3 2023-11-19T01:26:18,512 DEBUG [RS:0;jenkins-hbase6:44997 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/jenkins-hbase6:0, corePoolSize=3, maxPoolSize=3 2023-11-19T01:26:18,513 INFO [RS:0;jenkins-hbase6:44997 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2023-11-19T01:26:18,513 INFO [RS:0;jenkins-hbase6:44997 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2023-11-19T01:26:18,513 INFO [RS:0;jenkins-hbase6:44997 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2023-11-19T01:26:18,513 INFO [RS:0;jenkins-hbase6:44997 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2023-11-19T01:26:18,514 INFO [RS:0;jenkins-hbase6:44997 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,44997,1700357176784-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-19T01:26:18,529 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2023-11-19T01:26:18,533 INFO [RS:0;jenkins-hbase6:44997 {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,44997,1700357176784-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-19T01:26:18,554 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.Replication(204): jenkins-hbase6.apache.org,44997,1700357176784 started 2023-11-19T01:26:18,554 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1747): Serving as jenkins-hbase6.apache.org,44997,1700357176784, RpcServer on jenkins-hbase6.apache.org/172.31.2.20:44997, sessionid=0x10123a379ab0001 2023-11-19T01:26:18,554 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2023-11-19T01:26:18,554 DEBUG [RS:0;jenkins-hbase6:44997 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:18,554 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase6.apache.org,44997,1700357176784' 2023-11-19T01:26:18,554 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/flush-table-proc/abort' 2023-11-19T01:26:18,555 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/flush-table-proc/acquired' 2023-11-19T01:26:18,556 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2023-11-19T01:26:18,556 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.RegionServerProcedureManagerHost(51): Procedure backup-proc starting 2023-11-19T01:26:18,556 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase6.apache.org,44997,1700357176784' 2023-11-19T01:26:18,556 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:26:18,557 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:26:18,557 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.LogRollRegionServerProcedureManager(92): Started region server backup manager. 2023-11-19T01:26:18,557 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.RegionServerProcedureManagerHost(53): Procedure backup-proc started 2023-11-19T01:26:18,558 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2023-11-19T01:26:18,558 DEBUG [RS:0;jenkins-hbase6:44997 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:18,558 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'jenkins-hbase6.apache.org,44997,1700357176784' 2023-11-19T01:26:18,558 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/online-snapshot/abort' 2023-11-19T01:26:18,558 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/online-snapshot/acquired' 2023-11-19T01:26:18,559 DEBUG [RS:0;jenkins-hbase6:44997 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2023-11-19T01:26:18,559 INFO [RS:0;jenkins-hbase6:44997 {}] quotas.RegionServerRpcQuotaManager(63): Quota support disabled 2023-11-19T01:26:18,559 INFO [RS:0;jenkins-hbase6:44997 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2023-11-19T01:26:18,580 WARN [jenkins-hbase6:44889 {}] assignment.AssignmentManager(2298): No servers available; cannot place 1 unassigned regions. 2023-11-19T01:26:18,670 INFO [RS:0;jenkins-hbase6:44997 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2023-11-19T01:26:18,673 INFO [RS:0;jenkins-hbase6:44997 {}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase6.apache.org%2C44997%2C1700357176784, suffix=, logDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784, archiveDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/oldWALs, maxLogs=32 2023-11-19T01:26:18,687 DEBUG [RS:0;jenkins-hbase6:44997 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357178676, exclude list is [], retry=0 2023-11-19T01:26:18,693 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:26:18,697 INFO [RS:0;jenkins-hbase6:44997 {}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357178676 2023-11-19T01:26:18,697 DEBUG [RS:0;jenkins-hbase6:44997 {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:26:18,832 DEBUG [jenkins-hbase6:44889 {}] assignment.AssignmentManager(2319): Processing assignQueue; systemServersCount=1, allServersCount=1 2023-11-19T01:26:18,841 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:18,846 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase6.apache.org,44997,1700357176784, state=OPENING 2023-11-19T01:26:18,855 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2023-11-19T01:26:18,856 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:26:18,856 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:26:18,857 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-19T01:26:18,857 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-19T01:26:18,862 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:26:19,222 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:19,226 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=AdminService, sasl=false 2023-11-19T01:26:19,231 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:40954, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2023-11-19T01:26:19,247 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(130): Open hbase:meta,,1.1588230740 2023-11-19T01:26:19,249 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2023-11-19T01:26:19,249 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2023-11-19T01:26:19,253 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(491): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta, suffix=.meta, logDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784, archiveDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/oldWALs, maxLogs=32 2023-11-19T01:26:19,272 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357179255.meta, exclude list is [], retry=0 2023-11-19T01:26:19,278 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:26:19,285 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(832): New WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357179255.meta 2023-11-19T01:26:19,286 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:26:19,287 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7254): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:26:19,294 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:26:19,295 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2023-11-19T01:26:19,319 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7955): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2023-11-19T01:26:19,319 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(393): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2023-11-19T01:26:19,326 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2023-11-19T01:26:19,326 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(888): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:19,327 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7296): checking encryption for 1588230740 2023-11-19T01:26:19,327 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7299): checking classloading for 1588230740 2023-11-19T01:26:19,330 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2023-11-19T01:26:19,332 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2023-11-19T01:26:19,333 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:19,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:26:19,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2023-11-19T01:26:19,337 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2023-11-19T01:26:19,337 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:19,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:26:19,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2023-11-19T01:26:19,339 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2023-11-19T01:26:19,339 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:19,340 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(324): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2023-11-19T01:26:19,342 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740 2023-11-19T01:26:19,345 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740 2023-11-19T01:26:19,349 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2023-11-19T01:26:19,352 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1076): writing seq id for 1588230740 2023-11-19T01:26:19,354 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11197367040, jitterRate=0.04283607006072998}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2023-11-19T01:26:19,355 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(992): Region open journal for 1588230740: 2023-11-19T01:26:19,372 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1700357179209 2023-11-19T01:26:19,399 DEBUG [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:meta,,1.1588230740 2023-11-19T01:26:19,400 INFO [RS_OPEN_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(158): Opened hbase:meta,,1.1588230740 2023-11-19T01:26:19,401 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:19,404 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as jenkins-hbase6.apache.org,44997,1700357176784, state=OPEN 2023-11-19T01:26:19,408 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-11-19T01:26:19,408 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/meta-region-server 2023-11-19T01:26:19,409 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-19T01:26:19,409 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /1/meta-region-server: CHANGED 2023-11-19T01:26:19,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=3, resume processing ppid=2 2023-11-19T01:26:19,414 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=jenkins-hbase6.apache.org,44997,1700357176784 in 547 msec 2023-11-19T01:26:19,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=2, resume processing ppid=1 2023-11-19T01:26:19,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0090 sec 2023-11-19T01:26:19,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.3310 sec 2023-11-19T01:26:19,426 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1064): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1700357179426, completionTime=-1 2023-11-19T01:26:19,426 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ServerManager(816): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2023-11-19T01:26:19,427 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] assignment.AssignmentManager(1645): Joining cluster... 2023-11-19T01:26:19,524 DEBUG [hconnection-0x7ab9f512-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:26:19,528 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:40960, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:26:19,544 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] assignment.AssignmentManager(1657): Number of RegionServers=1 2023-11-19T01:26:19,544 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1700357239544 2023-11-19T01:26:19,544 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1700357299544 2023-11-19T01:26:19,545 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] assignment.AssignmentManager(1664): Joined the cluster in 117 msec 2023-11-19T01:26:19,570 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,44889,1700357173572-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2023-11-19T01:26:19,570 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,44889,1700357173572-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-19T01:26:19,570 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,44889,1700357173572-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2023-11-19T01:26:19,572 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-jenkins-hbase6:44889, period=300000, unit=MILLISECONDS is enabled. 2023-11-19T01:26:19,573 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2023-11-19T01:26:19,580 DEBUG [master/jenkins-hbase6:0.Chore.1 {}] janitor.CatalogJanitor(175): 2023-11-19T01:26:19,601 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2023-11-19T01:26:19,603 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(2369): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2023-11-19T01:26:19,616 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2023-11-19T01:26:19,621 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:26:19,624 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:19,626 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:26:19,648 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => f8f5111e949766749b5234926ea710eb, NAME => 'hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:26:19,660 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(888): Instantiated hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:19,660 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1668): Closing f8f5111e949766749b5234926ea710eb, disabling compactions & flushes 2023-11-19T01:26:19,660 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1690): Closing region hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb. 2023-11-19T01:26:19,660 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb. 2023-11-19T01:26:19,660 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb. after waiting 0 ms 2023-11-19T01:26:19,660 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb. 2023-11-19T01:26:19,660 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1909): Closed hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb. 2023-11-19T01:26:19,660 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1622): Region close journal for f8f5111e949766749b5234926ea710eb: 2023-11-19T01:26:19,663 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:26:19,679 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1700357179664"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357179664"}]},"ts":"1700357179664"} 2023-11-19T01:26:19,718 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:26:19,720 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:26:19,725 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357179720"}]},"ts":"1700357179720"} 2023-11-19T01:26:19,730 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2023-11-19T01:26:19,740 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=f8f5111e949766749b5234926ea710eb, ASSIGN}] 2023-11-19T01:26:19,743 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=f8f5111e949766749b5234926ea710eb, ASSIGN 2023-11-19T01:26:19,745 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=f8f5111e949766749b5234926ea710eb, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,44997,1700357176784; forceNewPlan=false, retain=false 2023-11-19T01:26:19,895 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=f8f5111e949766749b5234926ea710eb, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:19,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure f8f5111e949766749b5234926ea710eb, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:26:20,055 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:20,061 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(130): Open hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb. 2023-11-19T01:26:20,062 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7254): Opening region: {ENCODED => f8f5111e949766749b5234926ea710eb, NAME => 'hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:26:20,063 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:26:20,063 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace f8f5111e949766749b5234926ea710eb 2023-11-19T01:26:20,064 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(888): Instantiated hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:20,064 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7296): checking encryption for f8f5111e949766749b5234926ea710eb 2023-11-19T01:26:20,064 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7299): checking classloading for f8f5111e949766749b5234926ea710eb 2023-11-19T01:26:20,066 INFO [StoreOpener-f8f5111e949766749b5234926ea710eb-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region f8f5111e949766749b5234926ea710eb 2023-11-19T01:26:20,068 INFO [StoreOpener-f8f5111e949766749b5234926ea710eb-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f8f5111e949766749b5234926ea710eb columnFamilyName info 2023-11-19T01:26:20,068 DEBUG [StoreOpener-f8f5111e949766749b5234926ea710eb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:20,069 INFO [StoreOpener-f8f5111e949766749b5234926ea710eb-1 {}] regionserver.HStore(324): Store=f8f5111e949766749b5234926ea710eb/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:26:20,071 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/namespace/f8f5111e949766749b5234926ea710eb 2023-11-19T01:26:20,072 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/namespace/f8f5111e949766749b5234926ea710eb 2023-11-19T01:26:20,075 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1076): writing seq id for f8f5111e949766749b5234926ea710eb 2023-11-19T01:26:20,079 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/namespace/f8f5111e949766749b5234926ea710eb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:26:20,080 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1093): Opened f8f5111e949766749b5234926ea710eb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10575332480, jitterRate=-0.015095412731170654}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:26:20,081 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(992): Region open journal for f8f5111e949766749b5234926ea710eb: 2023-11-19T01:26:20,084 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2556): Post open deploy tasks for hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb., pid=6, masterSystemTime=1700357180054 2023-11-19T01:26:20,090 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2583): Finished post open deploy task for hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb. 2023-11-19T01:26:20,090 INFO [RS_OPEN_PRIORITY_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(158): Opened hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb. 2023-11-19T01:26:20,090 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=f8f5111e949766749b5234926ea710eb, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:20,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=6, resume processing ppid=5 2023-11-19T01:26:20,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure f8f5111e949766749b5234926ea710eb, server=jenkins-hbase6.apache.org,44997,1700357176784 in 198 msec 2023-11-19T01:26:20,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=5, resume processing ppid=4 2023-11-19T01:26:20,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=f8f5111e949766749b5234926ea710eb, ASSIGN in 363 msec 2023-11-19T01:26:20,109 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:26:20,109 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357180109"}]},"ts":"1700357180109"} 2023-11-19T01:26:20,113 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2023-11-19T01:26:20,116 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:26:20,120 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 513 msec 2023-11-19T01:26:20,121 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/namespace 2023-11-19T01:26:20,123 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:26:20,123 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/namespace 2023-11-19T01:26:20,123 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1 2023-11-19T01:26:20,187 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2023-11-19T01:26:20,209 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:26:20,216 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 38 msec 2023-11-19T01:26:20,225 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1032): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2023-11-19T01:26:20,238 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:26:20,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 16 msec 2023-11-19T01:26:20,253 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/default 2023-11-19T01:26:20,255 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/1/namespace/hbase 2023-11-19T01:26:20,258 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1194): Master has completed initialization 3.335sec 2023-11-19T01:26:20,260 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2023-11-19T01:26:20,262 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2023-11-19T01:26:20,263 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2023-11-19T01:26:20,264 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2023-11-19T01:26:20,264 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2023-11-19T01:26:20,265 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,44889,1700357173572-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2023-11-19T01:26:20,266 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=jenkins-hbase6.apache.org,44889,1700357173572-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2023-11-19T01:26:20,280 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster {}] master.HMaster(1297): Balancer post startup initialization complete, took 0 seconds 2023-11-19T01:26:20,281 INFO [master/jenkins-hbase6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2023-11-19T01:26:20,331 DEBUG [Listener at localhost/35765 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6f95b0aa to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:26:20,337 DEBUG [Listener at localhost/35765 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1176545f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:26:20,344 DEBUG [Listener at localhost/35765 {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2023-11-19T01:26:20,344 DEBUG [Listener at localhost/35765 {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2023-11-19T01:26:20,360 DEBUG [hconnection-0x1d999fb5-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:26:20,372 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:39180, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:26:20,383 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=jenkins-hbase6.apache.org,44889,1700357173572 2023-11-19T01:26:20,384 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(2786): Starting mini mapreduce cluster... 2023-11-19T01:26:20,384 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/test.cache.data in system properties and HBase conf 2023-11-19T01:26:20,384 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop.tmp.dir in system properties and HBase conf 2023-11-19T01:26:20,384 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop.log.dir in system properties and HBase conf 2023-11-19T01:26:20,384 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir in system properties and HBase conf 2023-11-19T01:26:20,384 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.temp.dir in system properties and HBase conf 2023-11-19T01:26:20,384 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2023-11-19T01:26:20,384 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:26:20,384 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2023-11-19T01:26:20,384 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/yarn.nodemanager.log-dirs in system properties and HBase conf 2023-11-19T01:26:20,384 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:26:20,385 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2023-11-19T01:26:20,385 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2023-11-19T01:26:20,385 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2023-11-19T01:26:20,385 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:26:20,385 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2023-11-19T01:26:20,385 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/nfs.dump.dir in system properties and HBase conf 2023-11-19T01:26:20,385 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/java.io.tmpdir in system properties and HBase conf 2023-11-19T01:26:20,385 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/dfs.journalnode.edits.dir in system properties and HBase conf 2023-11-19T01:26:20,385 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2023-11-19T01:26:20,385 INFO [Listener at localhost/35765 {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2023-11-19T01:26:21,726 WARN [Thread-249 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:26:22,103 INFO [Thread-249 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:26:22,109 INFO [Thread-249 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/jobhistory to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/java.io.tmpdir/Jetty_jenkins.hbase6_apache_org_33919_jobhistory____dn4p9k/webapp 2023-11-19T01:26:22,270 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices as a root resource class 2023-11-19T01:26:22,270 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver as a provider class 2023-11-19T01:26:22,270 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-19T01:26:22,274 INFO [Thread-249 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-19T01:26:22,361 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:26:22,485 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:26:22,784 INFO [Thread-249 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.mapreduce.v2.hs.webapp.HsWebServices to GuiceManagedComponentProvider with the scope "PerRequest" 2023-11-19T01:26:22,822 INFO [Thread-249 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:33919 2023-11-19T01:26:23,222 WARN [Listener at jenkins-hbase6.apache.org/38959 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:26:23,305 WARN [Listener at jenkins-hbase6.apache.org/38959 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-19T01:26:23,306 INFO [Listener at jenkins-hbase6.apache.org/38959 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:26:23,312 INFO [Listener at jenkins-hbase6.apache.org/38959 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/cluster to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/java.io.tmpdir/Jetty_jenkins.hbase6_apache_org_40559_cluster____dy1yax/webapp 2023-11-19T01:26:23,387 INFO [Listener at jenkins-hbase6.apache.org/38959 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver as a provider class 2023-11-19T01:26:23,387 INFO [Listener at jenkins-hbase6.apache.org/38959 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices as a root resource class 2023-11-19T01:26:23,387 INFO [Listener at jenkins-hbase6.apache.org/38959 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-19T01:26:23,387 INFO [Listener at jenkins-hbase6.apache.org/38959 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-19T01:26:23,398 INFO [Listener at jenkins-hbase6.apache.org/38959 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:26:23,641 INFO [Listener at jenkins-hbase6.apache.org/38959 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:26:23,722 INFO [Listener at jenkins-hbase6.apache.org/38959 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:26:23,729 INFO [Listener at jenkins-hbase6.apache.org/38959 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:40559 2023-11-19T01:26:24,102 WARN [Listener at jenkins-hbase6.apache.org/42893 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:26:24,125 WARN [Listener at jenkins-hbase6.apache.org/42893 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-19T01:26:24,125 INFO [Listener at jenkins-hbase6.apache.org/42893 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:26:24,132 INFO [Listener at jenkins-hbase6.apache.org/42893 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/java.io.tmpdir/Jetty_jenkins.hbase6_apache_org_33487_node____.av3stt/webapp 2023-11-19T01:26:24,184 INFO [Listener at jenkins-hbase6.apache.org/42893 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-19T01:26:24,184 INFO [Listener at jenkins-hbase6.apache.org/42893 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-19T01:26:24,184 INFO [Listener at jenkins-hbase6.apache.org/42893 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-19T01:26:24,184 INFO [Listener at jenkins-hbase6.apache.org/42893 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-19T01:26:24,192 INFO [Listener at jenkins-hbase6.apache.org/42893 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:26:24,215 INFO [Listener at jenkins-hbase6.apache.org/42893 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:26:24,291 INFO [Listener at jenkins-hbase6.apache.org/42893 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:26:24,300 INFO [Listener at jenkins-hbase6.apache.org/42893 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:33487 2023-11-19T01:26:24,464 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] http.HttpRequestLog(97): Jetty request log can only be enabled using Log4j 2023-11-19T01:26:24,474 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] servlet.GuiceFilter(87): Multiple Servlet injectors detected. This is a warning indicating that you have more than one GuiceFilter running in your web application. If this is deliberate, you may safely ignore this message. If this is NOT deliberate however, your application may not work as expected. 2023-11-19T01:26:24,474 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] log.Slf4jLog(67): jetty-6.1.26 2023-11-19T01:26:24,483 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] log.Slf4jLog(67): Extract jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-yarn-common/2.10.2/hadoop-yarn-common-2.10.2.jar!/webapps/node to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/java.io.tmpdir/Jetty_jenkins.hbase6_apache_org_39401_node____.5snyid/webapp 2023-11-19T01:26:24,552 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] container.GuiceComponentProviderFactory(107): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices as a root resource class 2023-11-19T01:26:24,552 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.webapp.GenericExceptionHandler as a provider class 2023-11-19T01:26:24,552 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] container.GuiceComponentProviderFactory(104): Registering org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver as a provider class 2023-11-19T01:26:24,553 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] application.WebApplicationImpl(791): Initiating Jersey application, version 'Jersey: 1.9 09/02/2011 11:17 AM' 2023-11-19T01:26:24,562 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.JAXBContextResolver to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:26:24,579 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.webapp.GenericExceptionHandler to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:26:24,641 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] container.GuiceComponentProviderFactory(159): Binding org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices to GuiceManagedComponentProvider with the scope "Singleton" 2023-11-19T01:26:24,647 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] log.Slf4jLog(67): Started HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:39401 2023-11-19T01:26:24,679 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.HBaseTestingUtility(2821): Mini mapreduce cluster started 2023-11-19T01:26:24,679 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestBackupBase(301): ROOTDIR hdfs://localhost:42739/backupUT 2023-11-19T01:26:24,690 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:26:24,694 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:47296, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:26:24,703 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$18(3398): Client=jenkins//172.31.2.20 creating {NAME => 'ns1'} 2023-11-19T01:26:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=9, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns1 2023-11-19T01:26:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2023-11-19T01:26:24,735 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:26:24,740 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=9, state=SUCCESS; CreateNamespaceProcedure, namespace=ns1 in 35 msec 2023-11-19T01:26:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=9 2023-11-19T01:26:24,825 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$18(3398): Client=jenkins//172.31.2.20 creating {NAME => 'ns2'} 2023-11-19T01:26:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=10, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns2 2023-11-19T01:26:24,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=10 2023-11-19T01:26:24,838 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:26:24,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=10, state=SUCCESS; CreateNamespaceProcedure, namespace=ns2 in 18 msec 2023-11-19T01:26:24,878 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:26:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=10 2023-11-19T01:26:24,935 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$18(3398): Client=jenkins//172.31.2.20 creating {NAME => 'ns3'} 2023-11-19T01:26:24,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=11, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns3 2023-11-19T01:26:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=11 2023-11-19T01:26:24,950 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:26:24,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=11, state=SUCCESS; CreateNamespaceProcedure, namespace=ns3 in 18 msec 2023-11-19T01:26:24,969 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2023-11-19T01:26:24,970 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2023-11-19T01:26:24,971 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2023-11-19T01:26:24,971 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2023-11-19T01:26:24,973 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2023-11-19T01:26:24,973 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2023-11-19T01:26:24,974 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:26:24,974 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.backup.BackupObserver Metrics about HBase RegionObservers 2023-11-19T01:26:24,974 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-19T01:26:24,974 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2023-11-19T01:26:24,974 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2023-11-19T01:26:24,974 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2023-11-19T01:26:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=11 2023-11-19T01:26:25,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$18(3398): Client=jenkins//172.31.2.20 creating {NAME => 'ns4'} 2023-11-19T01:26:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=12, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=ns4 2023-11-19T01:26:25,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2023-11-19T01:26:25,059 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:26:25,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=12, state=SUCCESS; CreateNamespaceProcedure, namespace=ns4 in 16 msec 2023-11-19T01:26:25,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=12 2023-11-19T01:26:25,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'test-1700357184679', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:26:25,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=test-1700357184679 2023-11-19T01:26:25,172 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=test-1700357184679 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:26:25,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "default" qualifier: "test-1700357184679" procId is: 13 2023-11-19T01:26:25,173 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:25,174 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=test-1700357184679 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:26:25,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:26:25,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:26:25,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:26:25,611 INFO [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 2c9c20d76de40bb8afca0e6f5e27adb8, NAME => 'test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1700357184679', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:26:25,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:26:26,029 DEBUG [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(888): Instantiated test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:26,029 DEBUG [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1668): Closing 2c9c20d76de40bb8afca0e6f5e27adb8, disabling compactions & flushes 2023-11-19T01:26:26,029 INFO [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1690): Closing region test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8. 2023-11-19T01:26:26,029 DEBUG [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8. 2023-11-19T01:26:26,029 DEBUG [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8. after waiting 0 ms 2023-11-19T01:26:26,029 DEBUG [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8. 2023-11-19T01:26:26,029 INFO [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1909): Closed test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8. 2023-11-19T01:26:26,029 DEBUG [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1622): Region close journal for 2c9c20d76de40bb8afca0e6f5e27adb8: 2023-11-19T01:26:26,031 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=test-1700357184679 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:26:26,031 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1700357186031"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357186031"}]},"ts":"1700357186031"} 2023-11-19T01:26:26,034 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:26:26,035 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=test-1700357184679 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:26:26,035 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1700357184679","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357186035"}]},"ts":"1700357186035"} 2023-11-19T01:26:26,037 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1700357184679, state=ENABLING in hbase:meta 2023-11-19T01:26:26,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1700357184679, region=2c9c20d76de40bb8afca0e6f5e27adb8, ASSIGN}] 2023-11-19T01:26:26,044 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1700357184679, region=2c9c20d76de40bb8afca0e6f5e27adb8, ASSIGN 2023-11-19T01:26:26,046 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=14, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1700357184679, region=2c9c20d76de40bb8afca0e6f5e27adb8, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,44997,1700357176784; forceNewPlan=false, retain=false 2023-11-19T01:26:26,197 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=2c9c20d76de40bb8afca0e6f5e27adb8, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:26,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 2c9c20d76de40bb8afca0e6f5e27adb8, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:26:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:26:26,353 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:26,361 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(130): Open test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8. 2023-11-19T01:26:26,362 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7254): Opening region: {ENCODED => 2c9c20d76de40bb8afca0e6f5e27adb8, NAME => 'test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:26:26,362 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:26:26,362 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1700357184679 2c9c20d76de40bb8afca0e6f5e27adb8 2023-11-19T01:26:26,362 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(888): Instantiated test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:26,363 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7296): checking encryption for 2c9c20d76de40bb8afca0e6f5e27adb8 2023-11-19T01:26:26,363 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(7299): checking classloading for 2c9c20d76de40bb8afca0e6f5e27adb8 2023-11-19T01:26:26,365 INFO [StoreOpener-2c9c20d76de40bb8afca0e6f5e27adb8-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 2c9c20d76de40bb8afca0e6f5e27adb8 2023-11-19T01:26:26,368 INFO [StoreOpener-2c9c20d76de40bb8afca0e6f5e27adb8-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c9c20d76de40bb8afca0e6f5e27adb8 columnFamilyName f 2023-11-19T01:26:26,368 DEBUG [StoreOpener-2c9c20d76de40bb8afca0e6f5e27adb8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:26,369 INFO [StoreOpener-2c9c20d76de40bb8afca0e6f5e27adb8-1 {}] regionserver.HStore(324): Store=2c9c20d76de40bb8afca0e6f5e27adb8/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:26:26,370 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8 2023-11-19T01:26:26,371 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8 2023-11-19T01:26:26,377 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1076): writing seq id for 2c9c20d76de40bb8afca0e6f5e27adb8 2023-11-19T01:26:26,382 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:26:26,383 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(1093): Opened 2c9c20d76de40bb8afca0e6f5e27adb8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10557012320, jitterRate=-0.016801610589027405}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:26:26,384 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegion(992): Region open journal for 2c9c20d76de40bb8afca0e6f5e27adb8: 2023-11-19T01:26:26,385 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8., pid=15, masterSystemTime=1700357186353 2023-11-19T01:26:26,388 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8. 2023-11-19T01:26:26,388 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=15}] handler.AssignRegionHandler(158): Opened test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8. 2023-11-19T01:26:26,389 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=14 updating hbase:meta row=2c9c20d76de40bb8afca0e6f5e27adb8, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:26,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=15, resume processing ppid=14 2023-11-19T01:26:26,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 2c9c20d76de40bb8afca0e6f5e27adb8, server=jenkins-hbase6.apache.org,44997,1700357176784 in 191 msec 2023-11-19T01:26:26,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=14, resume processing ppid=13 2023-11-19T01:26:26,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=14, ppid=13, state=SUCCESS; TransitRegionStateProcedure table=test-1700357184679, region=2c9c20d76de40bb8afca0e6f5e27adb8, ASSIGN in 352 msec 2023-11-19T01:26:26,398 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=test-1700357184679 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:26:26,398 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1700357184679","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357186398"}]},"ts":"1700357186398"} 2023-11-19T01:26:26,400 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1700357184679, state=ENABLED in hbase:meta 2023-11-19T01:26:26,403 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=13, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=test-1700357184679 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:26:26,405 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=13, state=SUCCESS; CreateTableProcedure table=test-1700357184679 in 1.2350 sec 2023-11-19T01:26:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=13 2023-11-19T01:26:27,290 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: default:test-1700357184679, procId: 13 completed 2023-11-19T01:26:27,292 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x55722d8c to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:26:27,298 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67eb8cdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:26:27,301 DEBUG [hconnection-0x1bacd643-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:26:27,304 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:39182, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:26:27,309 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegion(8210): writing data to region test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8. with WAL disabled. Data may be lost in the event of a crash. 2023-11-19T01:26:27,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'ns2:test-17003571846791', {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:26:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:test-17003571846791 2023-11-19T01:26:27,408 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:test-17003571846791 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:26:27,408 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:27,408 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "ns2" qualifier: "test-17003571846791" procId is: 16 2023-11-19T01:26:27,409 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:test-17003571846791 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:26:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-19T01:26:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-19T01:26:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-19T01:26:27,825 INFO [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => dd0bac2484821b3c9f7478186b974ffd, NAME => 'ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-17003571846791', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:26:27,838 DEBUG [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:27,838 DEBUG [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1668): Closing dd0bac2484821b3c9f7478186b974ffd, disabling compactions & flushes 2023-11-19T01:26:27,838 INFO [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd. 2023-11-19T01:26:27,838 DEBUG [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd. 2023-11-19T01:26:27,838 DEBUG [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd. after waiting 0 ms 2023-11-19T01:26:27,838 DEBUG [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd. 2023-11-19T01:26:27,838 INFO [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd. 2023-11-19T01:26:27,839 DEBUG [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1622): Region close journal for dd0bac2484821b3c9f7478186b974ffd: 2023-11-19T01:26:27,840 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:test-17003571846791 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:26:27,840 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1700357187840"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357187840"}]},"ts":"1700357187840"} 2023-11-19T01:26:27,843 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:26:27,844 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:test-17003571846791 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:26:27,844 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-17003571846791","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357187844"}]},"ts":"1700357187844"} 2023-11-19T01:26:27,846 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-17003571846791, state=ENABLING in hbase:meta 2023-11-19T01:26:27,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-17003571846791, region=dd0bac2484821b3c9f7478186b974ffd, ASSIGN}] 2023-11-19T01:26:27,852 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-17003571846791, region=dd0bac2484821b3c9f7478186b974ffd, ASSIGN 2023-11-19T01:26:27,853 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=17, ppid=16, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-17003571846791, region=dd0bac2484821b3c9f7478186b974ffd, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,44997,1700357176784; forceNewPlan=false, retain=false 2023-11-19T01:26:28,004 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=dd0bac2484821b3c9f7478186b974ffd, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:28,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; OpenRegionProcedure dd0bac2484821b3c9f7478186b974ffd, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:26:28,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-19T01:26:28,159 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:28,163 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(130): Open ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd. 2023-11-19T01:26:28,163 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7254): Opening region: {ENCODED => dd0bac2484821b3c9f7478186b974ffd, NAME => 'ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:26:28,164 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:26:28,164 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17003571846791 dd0bac2484821b3c9f7478186b974ffd 2023-11-19T01:26:28,164 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(888): Instantiated ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:28,164 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7296): checking encryption for dd0bac2484821b3c9f7478186b974ffd 2023-11-19T01:26:28,164 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(7299): checking classloading for dd0bac2484821b3c9f7478186b974ffd 2023-11-19T01:26:28,166 INFO [StoreOpener-dd0bac2484821b3c9f7478186b974ffd-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region dd0bac2484821b3c9f7478186b974ffd 2023-11-19T01:26:28,168 INFO [StoreOpener-dd0bac2484821b3c9f7478186b974ffd-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd0bac2484821b3c9f7478186b974ffd columnFamilyName f 2023-11-19T01:26:28,168 DEBUG [StoreOpener-dd0bac2484821b3c9f7478186b974ffd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:28,168 INFO [StoreOpener-dd0bac2484821b3c9f7478186b974ffd-1 {}] regionserver.HStore(324): Store=dd0bac2484821b3c9f7478186b974ffd/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:26:28,169 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd 2023-11-19T01:26:28,170 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd 2023-11-19T01:26:28,173 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1076): writing seq id for dd0bac2484821b3c9f7478186b974ffd 2023-11-19T01:26:28,181 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:26:28,182 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(1093): Opened dd0bac2484821b3c9f7478186b974ffd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11565542240, jitterRate=0.07712505757808685}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:26:28,183 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegion(992): Region open journal for dd0bac2484821b3c9f7478186b974ffd: 2023-11-19T01:26:28,185 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd., pid=18, masterSystemTime=1700357188159 2023-11-19T01:26:28,187 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd. 2023-11-19T01:26:28,187 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=18}] handler.AssignRegionHandler(158): Opened ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd. 2023-11-19T01:26:28,188 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=17 updating hbase:meta row=dd0bac2484821b3c9f7478186b974ffd, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:28,193 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=18, resume processing ppid=17 2023-11-19T01:26:28,193 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=18, ppid=17, state=SUCCESS; OpenRegionProcedure dd0bac2484821b3c9f7478186b974ffd, server=jenkins-hbase6.apache.org,44997,1700357176784 in 184 msec 2023-11-19T01:26:28,195 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=17, resume processing ppid=16 2023-11-19T01:26:28,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=17, ppid=16, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-17003571846791, region=dd0bac2484821b3c9f7478186b974ffd, ASSIGN in 343 msec 2023-11-19T01:26:28,196 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:test-17003571846791 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:26:28,197 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-17003571846791","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357188196"}]},"ts":"1700357188196"} 2023-11-19T01:26:28,199 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-17003571846791, state=ENABLED in hbase:meta 2023-11-19T01:26:28,208 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=16, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:test-17003571846791 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:26:28,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=16, state=SUCCESS; CreateTableProcedure table=ns2:test-17003571846791 in 803 msec 2023-11-19T01:26:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=16 2023-11-19T01:26:28,517 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns2:test-17003571846791, procId: 16 completed 2023-11-19T01:26:28,521 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegion(8210): writing data to region ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd. with WAL disabled. Data may be lost in the event of a crash. 2023-11-19T01:26:28,625 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'ns3:test-17003571846792', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:26:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:test-17003571846792 2023-11-19T01:26:28,629 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:test-17003571846792 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:26:28,630 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:28,630 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "ns3" qualifier: "test-17003571846792" procId is: 19 2023-11-19T01:26:28,631 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:test-17003571846792 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:26:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:26:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:26:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:26:29,047 INFO [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 628b78cfbf31117825b170549256b565, NAME => 'ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-17003571846792', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:26:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:26:29,464 DEBUG [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:29,464 DEBUG [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1668): Closing 628b78cfbf31117825b170549256b565, disabling compactions & flushes 2023-11-19T01:26:29,465 INFO [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565. 2023-11-19T01:26:29,465 DEBUG [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565. 2023-11-19T01:26:29,465 DEBUG [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565. after waiting 0 ms 2023-11-19T01:26:29,465 DEBUG [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565. 2023-11-19T01:26:29,465 INFO [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565. 2023-11-19T01:26:29,465 DEBUG [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1622): Region close journal for 628b78cfbf31117825b170549256b565: 2023-11-19T01:26:29,466 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:test-17003571846792 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:26:29,467 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1700357189467"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357189467"}]},"ts":"1700357189467"} 2023-11-19T01:26:29,469 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:26:29,470 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:test-17003571846792 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:26:29,471 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-17003571846792","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357189470"}]},"ts":"1700357189470"} 2023-11-19T01:26:29,472 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-17003571846792, state=ENABLING in hbase:meta 2023-11-19T01:26:29,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-17003571846792, region=628b78cfbf31117825b170549256b565, ASSIGN}] 2023-11-19T01:26:29,482 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-17003571846792, region=628b78cfbf31117825b170549256b565, ASSIGN 2023-11-19T01:26:29,482 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=20, ppid=19, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-17003571846792, region=628b78cfbf31117825b170549256b565, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,44997,1700357176784; forceNewPlan=false, retain=false 2023-11-19T01:26:29,634 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=628b78cfbf31117825b170549256b565, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:29,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; OpenRegionProcedure 628b78cfbf31117825b170549256b565, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:26:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:26:29,789 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:29,794 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(130): Open ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565. 2023-11-19T01:26:29,794 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7254): Opening region: {ENCODED => 628b78cfbf31117825b170549256b565, NAME => 'ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:26:29,794 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:26:29,794 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17003571846792 628b78cfbf31117825b170549256b565 2023-11-19T01:26:29,794 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(888): Instantiated ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:29,794 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7296): checking encryption for 628b78cfbf31117825b170549256b565 2023-11-19T01:26:29,794 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7299): checking classloading for 628b78cfbf31117825b170549256b565 2023-11-19T01:26:29,796 INFO [StoreOpener-628b78cfbf31117825b170549256b565-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 628b78cfbf31117825b170549256b565 2023-11-19T01:26:29,798 INFO [StoreOpener-628b78cfbf31117825b170549256b565-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 628b78cfbf31117825b170549256b565 columnFamilyName f 2023-11-19T01:26:29,798 DEBUG [StoreOpener-628b78cfbf31117825b170549256b565-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:29,799 INFO [StoreOpener-628b78cfbf31117825b170549256b565-1 {}] regionserver.HStore(324): Store=628b78cfbf31117825b170549256b565/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:26:29,800 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/628b78cfbf31117825b170549256b565 2023-11-19T01:26:29,800 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/628b78cfbf31117825b170549256b565 2023-11-19T01:26:29,803 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1076): writing seq id for 628b78cfbf31117825b170549256b565 2023-11-19T01:26:29,806 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/628b78cfbf31117825b170549256b565/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:26:29,807 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): Opened 628b78cfbf31117825b170549256b565; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10994149760, jitterRate=0.02390998601913452}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:26:29,807 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(992): Region open journal for 628b78cfbf31117825b170549256b565: 2023-11-19T01:26:29,808 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565., pid=21, masterSystemTime=1700357189789 2023-11-19T01:26:29,810 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565. 2023-11-19T01:26:29,810 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(158): Opened ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565. 2023-11-19T01:26:29,811 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=20 updating hbase:meta row=628b78cfbf31117825b170549256b565, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:29,815 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=21, resume processing ppid=20 2023-11-19T01:26:29,815 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=21, ppid=20, state=SUCCESS; OpenRegionProcedure 628b78cfbf31117825b170549256b565, server=jenkins-hbase6.apache.org,44997,1700357176784 in 177 msec 2023-11-19T01:26:29,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=20, resume processing ppid=19 2023-11-19T01:26:29,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=20, ppid=19, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-17003571846792, region=628b78cfbf31117825b170549256b565, ASSIGN in 335 msec 2023-11-19T01:26:29,818 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:test-17003571846792 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:26:29,818 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-17003571846792","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357189818"}]},"ts":"1700357189818"} 2023-11-19T01:26:29,820 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-17003571846792, state=ENABLED in hbase:meta 2023-11-19T01:26:29,824 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=19, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:test-17003571846792 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:26:29,826 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=19, state=SUCCESS; CreateTableProcedure table=ns3:test-17003571846792 in 1.1980 sec 2023-11-19T01:26:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=19 2023-11-19T01:26:30,740 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns3:test-17003571846792, procId: 19 completed 2023-11-19T01:26:30,740 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns3:test-17003571846792 get assigned. Timeout = 60000ms 2023-11-19T01:26:30,742 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-19T01:26:30,747 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.HBaseTestingUtility(3581): All regions for table ns3:test-17003571846792 assigned to meta. Checking AM states. 2023-11-19T01:26:30,748 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-19T01:26:30,748 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.HBaseTestingUtility(3601): All regions for table ns3:test-17003571846792 assigned. 2023-11-19T01:26:30,750 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'ns4:test-17003571846793', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:26:30,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns4:test-17003571846793 2023-11-19T01:26:30,754 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns4:test-17003571846793 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:26:30,754 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:30,754 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "ns4" qualifier: "test-17003571846793" procId is: 22 2023-11-19T01:26:30,755 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns4:test-17003571846793 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:26:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:26:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:26:30,971 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-17003571846791' 2023-11-19T01:26:30,972 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:test-17003571846792' 2023-11-19T01:26:30,972 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test-1700357184679' 2023-11-19T01:26:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:26:31,169 INFO [RegionOpenAndInit-ns4:test-17003571846793-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => fddaaf63127dc05fabd03e1195f3962c, NAME => 'ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns4:test-17003571846793', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:26:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:26:31,584 DEBUG [RegionOpenAndInit-ns4:test-17003571846793-pool-0 {}] regionserver.HRegion(888): Instantiated ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:31,584 DEBUG [RegionOpenAndInit-ns4:test-17003571846793-pool-0 {}] regionserver.HRegion(1668): Closing fddaaf63127dc05fabd03e1195f3962c, disabling compactions & flushes 2023-11-19T01:26:31,584 INFO [RegionOpenAndInit-ns4:test-17003571846793-pool-0 {}] regionserver.HRegion(1690): Closing region ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c. 2023-11-19T01:26:31,584 DEBUG [RegionOpenAndInit-ns4:test-17003571846793-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c. 2023-11-19T01:26:31,584 DEBUG [RegionOpenAndInit-ns4:test-17003571846793-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c. after waiting 0 ms 2023-11-19T01:26:31,584 DEBUG [RegionOpenAndInit-ns4:test-17003571846793-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c. 2023-11-19T01:26:31,584 INFO [RegionOpenAndInit-ns4:test-17003571846793-pool-0 {}] regionserver.HRegion(1909): Closed ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c. 2023-11-19T01:26:31,584 DEBUG [RegionOpenAndInit-ns4:test-17003571846793-pool-0 {}] regionserver.HRegion(1622): Region close journal for fddaaf63127dc05fabd03e1195f3962c: 2023-11-19T01:26:31,586 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns4:test-17003571846793 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:26:31,586 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1700357191586"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357191586"}]},"ts":"1700357191586"} 2023-11-19T01:26:31,588 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:26:31,589 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns4:test-17003571846793 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:26:31,589 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-17003571846793","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357191589"}]},"ts":"1700357191589"} 2023-11-19T01:26:31,591 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-17003571846793, state=ENABLING in hbase:meta 2023-11-19T01:26:31,596 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-17003571846793, region=fddaaf63127dc05fabd03e1195f3962c, ASSIGN}] 2023-11-19T01:26:31,598 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns4:test-17003571846793, region=fddaaf63127dc05fabd03e1195f3962c, ASSIGN 2023-11-19T01:26:31,599 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns4:test-17003571846793, region=fddaaf63127dc05fabd03e1195f3962c, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,44997,1700357176784; forceNewPlan=false, retain=false 2023-11-19T01:26:31,750 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=fddaaf63127dc05fabd03e1195f3962c, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:31,753 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE; OpenRegionProcedure fddaaf63127dc05fabd03e1195f3962c, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:26:31,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:26:31,866 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:26:31,906 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:31,912 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(130): Open ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c. 2023-11-19T01:26:31,912 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7254): Opening region: {ENCODED => fddaaf63127dc05fabd03e1195f3962c, NAME => 'ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:26:31,912 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:26:31,912 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17003571846793 fddaaf63127dc05fabd03e1195f3962c 2023-11-19T01:26:31,913 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(888): Instantiated ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:31,913 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7296): checking encryption for fddaaf63127dc05fabd03e1195f3962c 2023-11-19T01:26:31,913 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(7299): checking classloading for fddaaf63127dc05fabd03e1195f3962c 2023-11-19T01:26:31,915 INFO [StoreOpener-fddaaf63127dc05fabd03e1195f3962c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region fddaaf63127dc05fabd03e1195f3962c 2023-11-19T01:26:31,917 INFO [StoreOpener-fddaaf63127dc05fabd03e1195f3962c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fddaaf63127dc05fabd03e1195f3962c columnFamilyName f 2023-11-19T01:26:31,917 DEBUG [StoreOpener-fddaaf63127dc05fabd03e1195f3962c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:31,918 INFO [StoreOpener-fddaaf63127dc05fabd03e1195f3962c-1 {}] regionserver.HStore(324): Store=fddaaf63127dc05fabd03e1195f3962c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:26:31,919 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns4/test-17003571846793/fddaaf63127dc05fabd03e1195f3962c 2023-11-19T01:26:31,919 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns4/test-17003571846793/fddaaf63127dc05fabd03e1195f3962c 2023-11-19T01:26:31,922 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1076): writing seq id for fddaaf63127dc05fabd03e1195f3962c 2023-11-19T01:26:31,925 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns4/test-17003571846793/fddaaf63127dc05fabd03e1195f3962c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:26:31,926 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(1093): Opened fddaaf63127dc05fabd03e1195f3962c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11602550240, jitterRate=0.08057169616222382}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:26:31,926 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegion(992): Region open journal for fddaaf63127dc05fabd03e1195f3962c: 2023-11-19T01:26:31,928 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2556): Post open deploy tasks for ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c., pid=24, masterSystemTime=1700357191906 2023-11-19T01:26:31,930 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] regionserver.HRegionServer(2583): Finished post open deploy task for ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c. 2023-11-19T01:26:31,930 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=24}] handler.AssignRegionHandler(158): Opened ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c. 2023-11-19T01:26:31,931 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=23 updating hbase:meta row=fddaaf63127dc05fabd03e1195f3962c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:31,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=24, resume processing ppid=23 2023-11-19T01:26:31,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=24, ppid=23, state=SUCCESS; OpenRegionProcedure fddaaf63127dc05fabd03e1195f3962c, server=jenkins-hbase6.apache.org,44997,1700357176784 in 180 msec 2023-11-19T01:26:31,938 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=23, resume processing ppid=22 2023-11-19T01:26:31,938 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=23, ppid=22, state=SUCCESS; TransitRegionStateProcedure table=ns4:test-17003571846793, region=fddaaf63127dc05fabd03e1195f3962c, ASSIGN in 339 msec 2023-11-19T01:26:31,939 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns4:test-17003571846793 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:26:31,939 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns4:test-17003571846793","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357191939"}]},"ts":"1700357191939"} 2023-11-19T01:26:31,941 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns4:test-17003571846793, state=ENABLED in hbase:meta 2023-11-19T01:26:31,944 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=22, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns4:test-17003571846793 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:26:31,946 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=22, state=SUCCESS; CreateTableProcedure table=ns4:test-17003571846793 in 1.1940 sec 2023-11-19T01:26:32,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=22 2023-11-19T01:26:32,863 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: ns4:test-17003571846793, procId: 22 completed 2023-11-19T01:26:32,863 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.HBaseTestingUtility(3527): Waiting until all regions of table ns4:test-17003571846793 get assigned. Timeout = 60000ms 2023-11-19T01:26:32,863 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-19T01:26:32,868 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.HBaseTestingUtility(3581): All regions for table ns4:test-17003571846793 assigned to meta. Checking AM states. 2023-11-19T01:26:32,868 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2023-11-19T01:26:32,868 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.HBaseTestingUtility(3601): All regions for table ns4:test-17003571846793 assigned. 2023-11-19T01:26:32,869 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x55722d8c to 127.0.0.1:57501 2023-11-19T01:26:32,869 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:26:32,910 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleOverwrite Thread=584, OpenFileDescriptor=689, MaxFileDescriptor=60000, SystemLoadAverage=211, ProcessCount=171, AvailableMemoryMB=4098 2023-11-19T01:26:32,910 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=584 is superior to 500 2023-11-19T01:26:32,913 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestFullRestore(216): create full backup image on multiple tables 2023-11-19T01:26:32,914 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x11ad338b to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:26:32,920 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cf8cc83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:26:32,929 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:26:32,930 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:39828, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:26:32,944 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:26:32,945 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:46598, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:26:32,955 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$18(3398): Client=jenkins//172.31.2.20 creating {NAME => 'backup'} 2023-11-19T01:26:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=25, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=backup 2023-11-19T01:26:32,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=25 2023-11-19T01:26:32,966 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/namespace 2023-11-19T01:26:32,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=25, state=SUCCESS; CreateNamespaceProcedure, namespace=backup in 14 msec 2023-11-19T01:26:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=25 2023-11-19T01:26:33,076 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'backup:system', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:26:33,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system 2023-11-19T01:26:33,081 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:26:33,081 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:33,081 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "backup" qualifier: "system" procId is: 26 2023-11-19T01:26:33,082 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:26:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-19T01:26:33,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-19T01:26:33,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-19T01:26:33,500 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 322466352cac3ade468ab2ce8e706acc, NAME => 'backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:26:33,512 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:33,512 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1668): Closing 322466352cac3ade468ab2ce8e706acc, disabling compactions & flushes 2023-11-19T01:26:33,512 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:26:33,512 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:26:33,512 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. after waiting 0 ms 2023-11-19T01:26:33,512 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:26:33,512 INFO [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1909): Closed backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:26:33,512 DEBUG [RegionOpenAndInit-backup:system-pool-0 {}] regionserver.HRegion(1622): Region close journal for 322466352cac3ade468ab2ce8e706acc: 2023-11-19T01:26:33,514 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:26:33,514 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc.","families":{"info":[{"qualifier":"regioninfo","vlen":39,"tag":[],"timestamp":"1700357193514"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357193514"}]},"ts":"1700357193514"} 2023-11-19T01:26:33,516 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:26:33,517 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:26:33,517 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357193517"}]},"ts":"1700357193517"} 2023-11-19T01:26:33,519 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLING in hbase:meta 2023-11-19T01:26:33,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=322466352cac3ade468ab2ce8e706acc, ASSIGN}] 2023-11-19T01:26:33,526 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system, region=322466352cac3ade468ab2ce8e706acc, ASSIGN 2023-11-19T01:26:33,527 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system, region=322466352cac3ade468ab2ce8e706acc, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,44997,1700357176784; forceNewPlan=false, retain=false 2023-11-19T01:26:33,678 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=322466352cac3ade468ab2ce8e706acc, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:33,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; OpenRegionProcedure 322466352cac3ade468ab2ce8e706acc, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:26:33,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-19T01:26:33,833 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:33,838 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(130): Open backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:26:33,838 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7254): Opening region: {ENCODED => 322466352cac3ade468ab2ce8e706acc, NAME => 'backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:26:33,839 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:26:33,839 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:26:33,839 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(888): Instantiated backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:33,839 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7296): checking encryption for 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:26:33,839 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7299): checking classloading for 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:26:33,841 INFO [StoreOpener-322466352cac3ade468ab2ce8e706acc-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:26:33,844 INFO [StoreOpener-322466352cac3ade468ab2ce8e706acc-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 322466352cac3ade468ab2ce8e706acc columnFamilyName meta 2023-11-19T01:26:33,844 DEBUG [StoreOpener-322466352cac3ade468ab2ce8e706acc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:33,845 INFO [StoreOpener-322466352cac3ade468ab2ce8e706acc-1 {}] regionserver.HStore(324): Store=322466352cac3ade468ab2ce8e706acc/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:26:33,845 INFO [StoreOpener-322466352cac3ade468ab2ce8e706acc-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:26:33,847 INFO [StoreOpener-322466352cac3ade468ab2ce8e706acc-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 322466352cac3ade468ab2ce8e706acc columnFamilyName session 2023-11-19T01:26:33,847 DEBUG [StoreOpener-322466352cac3ade468ab2ce8e706acc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:33,848 INFO [StoreOpener-322466352cac3ade468ab2ce8e706acc-1 {}] regionserver.HStore(324): Store=322466352cac3ade468ab2ce8e706acc/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:26:33,849 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc 2023-11-19T01:26:33,849 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc 2023-11-19T01:26:33,851 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-19T01:26:33,853 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1076): writing seq id for 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:26:33,855 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:26:33,856 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): Opened 322466352cac3ade468ab2ce8e706acc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11659132800, jitterRate=0.0858413577079773}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-19T01:26:33,857 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(992): Region open journal for 322466352cac3ade468ab2ce8e706acc: 2023-11-19T01:26:33,858 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc., pid=28, masterSystemTime=1700357193833 2023-11-19T01:26:33,860 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:26:33,860 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(158): Opened backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:26:33,861 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=27 updating hbase:meta row=322466352cac3ade468ab2ce8e706acc, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:33,867 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=28, resume processing ppid=27 2023-11-19T01:26:33,867 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=28, ppid=27, state=SUCCESS; OpenRegionProcedure 322466352cac3ade468ab2ce8e706acc, server=jenkins-hbase6.apache.org,44997,1700357176784 in 182 msec 2023-11-19T01:26:33,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=27, resume processing ppid=26 2023-11-19T01:26:33,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=27, ppid=26, state=SUCCESS; TransitRegionStateProcedure table=backup:system, region=322466352cac3ade468ab2ce8e706acc, ASSIGN in 343 msec 2023-11-19T01:26:33,870 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:26:33,870 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357193870"}]},"ts":"1700357193870"} 2023-11-19T01:26:33,872 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system, state=ENABLED in hbase:meta 2023-11-19T01:26:33,876 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:26:33,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=26, state=SUCCESS; CreateTableProcedure table=backup:system in 800 msec 2023-11-19T01:26:34,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=26 2023-11-19T01:26:34,191 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: backup:system, procId: 26 completed 2023-11-19T01:26:34,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'backup:system_bulk', {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:26:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=backup:system_bulk 2023-11-19T01:26:34,209 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:26:34,209 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:34,209 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "backup" qualifier: "system_bulk" procId is: 29 2023-11-19T01:26:34,210 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:26:34,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-19T01:26:34,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-19T01:26:34,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-19T01:26:34,623 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => e7af7e5272e1026697882da645f6c4df, NAME => 'backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='backup:system_bulk', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'meta', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'session', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:26:34,784 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system 2023-11-19T01:26:34,784 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system Metrics about Tables on a single HBase RegionServer 2023-11-19T01:26:34,785 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_test-17003571846791 2023-11-19T01:26:34,785 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_test-17003571846791 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:26:34,786 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17003571846792 2023-11-19T01:26:34,786 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-17003571846792 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:26:34,787 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns4_table_test-17003571846793 2023-11-19T01:26:34,787 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns4_table_test-17003571846793 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:26:34,788 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test-1700357184679 2023-11-19T01:26:34,788 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test-1700357184679 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:26:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-19T01:26:35,037 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:35,037 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1668): Closing e7af7e5272e1026697882da645f6c4df, disabling compactions & flushes 2023-11-19T01:26:35,037 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df. 2023-11-19T01:26:35,037 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df. 2023-11-19T01:26:35,037 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df. after waiting 0 ms 2023-11-19T01:26:35,037 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df. 2023-11-19T01:26:35,037 INFO [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1909): Closed backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df. 2023-11-19T01:26:35,037 DEBUG [RegionOpenAndInit-backup:system_bulk-pool-0 {}] regionserver.HRegion(1622): Region close journal for e7af7e5272e1026697882da645f6c4df: 2023-11-19T01:26:35,039 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:26:35,039 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1700357195039"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357195039"}]},"ts":"1700357195039"} 2023-11-19T01:26:35,041 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:26:35,042 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:26:35,042 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357195042"}]},"ts":"1700357195042"} 2023-11-19T01:26:35,044 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLING in hbase:meta 2023-11-19T01:26:35,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=e7af7e5272e1026697882da645f6c4df, ASSIGN}] 2023-11-19T01:26:35,051 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=backup:system_bulk, region=e7af7e5272e1026697882da645f6c4df, ASSIGN 2023-11-19T01:26:35,052 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=30, ppid=29, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=backup:system_bulk, region=e7af7e5272e1026697882da645f6c4df, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,44997,1700357176784; forceNewPlan=false, retain=false 2023-11-19T01:26:35,203 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=e7af7e5272e1026697882da645f6c4df, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:35,206 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; OpenRegionProcedure e7af7e5272e1026697882da645f6c4df, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:26:35,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-19T01:26:35,358 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:35,363 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(130): Open backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df. 2023-11-19T01:26:35,363 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7254): Opening region: {ENCODED => e7af7e5272e1026697882da645f6c4df, NAME => 'backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:26:35,363 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:26:35,363 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table system_bulk e7af7e5272e1026697882da645f6c4df 2023-11-19T01:26:35,363 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(888): Instantiated backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:35,364 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7296): checking encryption for e7af7e5272e1026697882da645f6c4df 2023-11-19T01:26:35,364 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7299): checking classloading for e7af7e5272e1026697882da645f6c4df 2023-11-19T01:26:35,365 INFO [StoreOpener-e7af7e5272e1026697882da645f6c4df-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family meta of region e7af7e5272e1026697882da645f6c4df 2023-11-19T01:26:35,367 INFO [StoreOpener-e7af7e5272e1026697882da645f6c4df-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7af7e5272e1026697882da645f6c4df columnFamilyName meta 2023-11-19T01:26:35,367 DEBUG [StoreOpener-e7af7e5272e1026697882da645f6c4df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:35,368 INFO [StoreOpener-e7af7e5272e1026697882da645f6c4df-1 {}] regionserver.HStore(324): Store=e7af7e5272e1026697882da645f6c4df/meta, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:26:35,368 INFO [StoreOpener-e7af7e5272e1026697882da645f6c4df-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family session of region e7af7e5272e1026697882da645f6c4df 2023-11-19T01:26:35,370 INFO [StoreOpener-e7af7e5272e1026697882da645f6c4df-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e7af7e5272e1026697882da645f6c4df columnFamilyName session 2023-11-19T01:26:35,370 DEBUG [StoreOpener-e7af7e5272e1026697882da645f6c4df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:35,371 INFO [StoreOpener-e7af7e5272e1026697882da645f6c4df-1 {}] regionserver.HStore(324): Store=e7af7e5272e1026697882da645f6c4df/session, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:26:35,371 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system_bulk/e7af7e5272e1026697882da645f6c4df 2023-11-19T01:26:35,372 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system_bulk/e7af7e5272e1026697882da645f6c4df 2023-11-19T01:26:35,374 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table backup:system_bulk descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2023-11-19T01:26:35,376 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1076): writing seq id for e7af7e5272e1026697882da645f6c4df 2023-11-19T01:26:35,379 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system_bulk/e7af7e5272e1026697882da645f6c4df/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:26:35,379 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): Opened e7af7e5272e1026697882da645f6c4df; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11158820960, jitterRate=0.03924618661403656}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2023-11-19T01:26:35,380 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(992): Region open journal for e7af7e5272e1026697882da645f6c4df: 2023-11-19T01:26:35,381 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2556): Post open deploy tasks for backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df., pid=31, masterSystemTime=1700357195358 2023-11-19T01:26:35,384 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2583): Finished post open deploy task for backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df. 2023-11-19T01:26:35,384 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(158): Opened backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df. 2023-11-19T01:26:35,384 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=30 updating hbase:meta row=e7af7e5272e1026697882da645f6c4df, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:35,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=31, resume processing ppid=30 2023-11-19T01:26:35,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=31, ppid=30, state=SUCCESS; OpenRegionProcedure e7af7e5272e1026697882da645f6c4df, server=jenkins-hbase6.apache.org,44997,1700357176784 in 180 msec 2023-11-19T01:26:35,391 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=30, resume processing ppid=29 2023-11-19T01:26:35,391 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=30, ppid=29, state=SUCCESS; TransitRegionStateProcedure table=backup:system_bulk, region=e7af7e5272e1026697882da645f6c4df, ASSIGN in 339 msec 2023-11-19T01:26:35,392 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:26:35,392 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"backup:system_bulk","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357195392"}]},"ts":"1700357195392"} 2023-11-19T01:26:35,394 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=backup:system_bulk, state=ENABLED in hbase:meta 2023-11-19T01:26:35,397 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=29, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=backup:system_bulk execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:26:35,399 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=29, state=SUCCESS; CreateTableProcedure table=backup:system_bulk in 1.1920 sec 2023-11-19T01:26:36,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=29 2023-11-19T01:26:36,318 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$TableFuture(3745): Operation: CREATE, Table Name: backup:system_bulk, procId: 29 completed 2023-11-19T01:26:36,331 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:26:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.BinaryComparator in 0ms 2023-11-19T01:26:36,366 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:26:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357196367 (current time:1700357196367). 2023-11-19T01:26:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:26:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:26:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3be3a63b to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:26:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5543ff5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:26:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:26:36,379 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:39842, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:26:36,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3be3a63b to 127.0.0.1:57501 2023-11-19T01:26:36,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:26:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:26:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=32, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:26:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-11-19T01:26:36,396 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:26:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-19T01:26:36,403 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:26:36,419 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:26:36,422 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:26:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-19T01:26:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-19T01:26:36,838 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:26:36,841 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc}] 2023-11-19T01:26:36,849 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:26:37,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-19T01:26:37,009 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:37,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=33 2023-11-19T01:26:37,011 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:26:37,013 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2819): Flushing 322466352cac3ade468ab2ce8e706acc 2/2 column families, dataSize=45 B heapSize=632 B 2023-11-19T01:26:37,091 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/8670d77389e247fc9026582c1b208b99 is 49, key is activesession:/session:c/1700357196352/Put/seqid=0 2023-11-19T01:26:37,416 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:26:37,474 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns4:test-17003571846793' 2023-11-19T01:26:37,474 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system_bulk' 2023-11-19T01:26:37,475 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'backup:system' 2023-11-19T01:26:37,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-19T01:26:37,510 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/8670d77389e247fc9026582c1b208b99 2023-11-19T01:26:37,583 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/8670d77389e247fc9026582c1b208b99 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99 2023-11-19T01:26:37,592 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99, entries=1, sequenceid=5, filesize=4.9 K 2023-11-19T01:26:37,596 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(3022): Finished flush of dataSize ~45 B/45, heapSize ~360 B/360, currentSize=0 B/0 for 322466352cac3ade468ab2ce8e706acc in 583ms, sequenceid=5, compaction requested=false 2023-11-19T01:26:37,598 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.HRegion(2520): Flush status journal for 322466352cac3ade468ab2ce8e706acc: 2023-11-19T01:26:37,598 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. for snapshot_backup_system completed. 2023-11-19T01:26:37,599 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:26:37,605 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:26:37,610 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-19T01:26:37,610 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99] hfiles 2023-11-19T01:26:37,610 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99 for snapshot=snapshot_backup_system 2023-11-19T01:26:38,027 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:26:38,030 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=33 2023-11-19T01:26:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=33 2023-11-19T01:26:38,034 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:26:38,035 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=33, ppid=32, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:26:38,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=33, resume processing ppid=32 2023-11-19T01:26:38,039 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:26:38,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=33, ppid=32, state=SUCCESS; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc in 1.1950 sec 2023-11-19T01:26:38,040 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:26:38,043 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:26:38,044 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:26:38,046 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:26:38,470 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:26:38,484 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:26:38,485 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:26:38,487 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=32, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:26:38,487 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 32 2023-11-19T01:26:38,489 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=32, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=32, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.0980 sec 2023-11-19T01:26:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=32 2023-11-19T01:26:38,504 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 32 completed 2023-11-19T01:26:38,505 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(115): Backup backup_1700357192923 started at 1700357198505. 2023-11-19T01:26:38,526 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(119): Backup session backup_1700357192923 has been started. 2023-11-19T01:26:38,529 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-19T01:26:38,541 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:26:38,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:26:38,545 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:26:38,545 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:26:38,545 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:26:38,545 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:26:38,546 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-19T01:26:38,546 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:26:38,548 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-19T01:26:38,548 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:38,548 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:26:38,548 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:26:38,549 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:38,549 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:26:38,549 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:26:38,549 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-19T01:26:38,550 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:26:38,550 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:26:38,550 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:26:38,552 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:26:38,552 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:26:38,552 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:26:38,552 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:26:38,553 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:26:38,553 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:26:38,553 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:26:38,557 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:38,557 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:26:38,557 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:38,557 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:26:38,557 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:26:38,558 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-11-19T01:26:38,558 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:26:38,558 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:26:38,558 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:26:38,559 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:26:38,559 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:38,560 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:26:38,560 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:26:38,560 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@4a8fe01a[Count = 0] remaining members to acquire global barrier 2023-11-19T01:26:38,560 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:26:38,560 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:26:38,563 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-19T01:26:38,563 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:26:38,563 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:26:38,563 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:26:38,563 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:38,563 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:26:38,564 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:26:38,564 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:38,564 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357178676 highest: 1700357178676 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:38,565 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784:(num 1700357178676) roll requested 2023-11-19T01:26:38,581 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357198565, exclude list is [], retry=0 2023-11-19T01:26:38,588 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:26:38,592 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357178676 with entries=19, filesize=4.02 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357198565 2023-11-19T01:26:38,594 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:26:38,594 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357178676 is not closed yet, will try archiving it next time 2023-11-19T01:26:38,595 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta:.meta(num 1700357179255) roll requested 2023-11-19T01:26:38,608 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357198595.meta, exclude list is [], retry=0 2023-11-19T01:26:38,613 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:26:38,617 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357179255.meta with entries=36, filesize=11.13 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357198595.meta 2023-11-19T01:26:38,617 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:26:38,617 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357179255.meta is not closed yet, will try archiving it next time 2023-11-19T01:26:38,625 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 61 2023-11-19T01:26:38,625 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357198565 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:38,626 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:26:38,628 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:46604, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-19T01:26:38,649 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:26:38,649 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,44997,1700357176784' in zk 2023-11-19T01:26:38,652 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:38,652 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:26:38,652 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:26:38,652 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:38,652 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:26:38,652 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:26:38,653 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:26:38,654 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:26:38,654 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:26:38,655 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:26:38,655 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:38,656 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:26:38,656 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:26:38,657 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:38,658 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,44997,1700357176784': 2023-11-19T01:26:38,658 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:26:38,658 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:26:38,658 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:26:38,658 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:26:38,658 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:26:38,658 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:26:38,662 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-19T01:26:38,662 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-19T01:26:38,662 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-19T01:26:38,662 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:26:38,662 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:26:38,662 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:26:38,662 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:26:38,662 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:26:38,662 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:26:38,662 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:26:38,663 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:26:38,663 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:38,663 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:26:38,663 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:26:38,663 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:26:38,664 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:26:38,664 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:38,665 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:38,665 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:26:38,665 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:26:38,666 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:38,679 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:38,679 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-19T01:26:38,679 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-19T01:26:38,679 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:26:38,679 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:26:38,679 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-19T01:26:38,680 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-19T01:26:38,680 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:38,680 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-19T01:26:38,680 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-19T01:26:38,680 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-19T01:26:38,680 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:26:38,680 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:26:38,680 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:26:38,681 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:26:38,681 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:26:38,683 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin(2906): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-11-19T01:26:38,683 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin(2914): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-11-19T01:26:38,784 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin(2920): Getting current status of procedure from master... 2023-11-19T01:26:38,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-19T01:26:38,794 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357198794_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } 2023-11-19T01:26:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357198795 (current time:1700357198795). 2023-11-19T01:26:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:26:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357198794_ns2_test-17003571846791 VERSION not specified, setting to 2 2023-11-19T01:26:38,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x21a4604e to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:26:38,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d1ea2ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:26:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:26:38,806 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:39856, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:26:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x21a4604e to 127.0.0.1:57501 2023-11-19T01:26:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:26:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:26:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=34, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357198794_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } 2023-11-19T01:26:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357198794_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-11-19T01:26:38,811 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357198794_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:26:38,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:26:38,812 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357198794_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:26:38,816 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357198794_ns2_test-17003571846791 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:26:38,818 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357198794_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:26:38,828 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357198794_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:26:38,828 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure dd0bac2484821b3c9f7478186b974ffd}] 2023-11-19T01:26:38,830 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure dd0bac2484821b3c9f7478186b974ffd 2023-11-19T01:26:38,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:26:38,981 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:38,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=35 2023-11-19T01:26:38,982 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd. 2023-11-19T01:26:38,983 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2819): Flushing dd0bac2484821b3c9f7478186b974ffd 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-11-19T01:26:39,003 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd/.tmp/f/05113e00edc94deeb1d57cf4e2bde358 is 37, key is row10/f:q1/1700357188531/Put/seqid=0 2023-11-19T01:26:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:26:39,413 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd/.tmp/f/05113e00edc94deeb1d57cf4e2bde358 2023-11-19T01:26:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:26:39,425 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd/.tmp/f/05113e00edc94deeb1d57cf4e2bde358 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd/f/05113e00edc94deeb1d57cf4e2bde358 2023-11-19T01:26:39,435 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd/f/05113e00edc94deeb1d57cf4e2bde358, entries=99, sequenceid=103, filesize=8.2 K 2023-11-19T01:26:39,436 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for dd0bac2484821b3c9f7478186b974ffd in 453ms, sequenceid=103, compaction requested=false 2023-11-19T01:26:39,437 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.HRegion(2520): Flush status journal for dd0bac2484821b3c9f7478186b974ffd: 2023-11-19T01:26:39,437 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd. for snapshot_1700357198794_ns2_test-17003571846791 completed. 2023-11-19T01:26:39,437 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(240): Storing 'ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd.' region-info for snapshot=snapshot_1700357198794_ns2_test-17003571846791 2023-11-19T01:26:39,437 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:26:39,437 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd/f/05113e00edc94deeb1d57cf4e2bde358] hfiles 2023-11-19T01:26:39,437 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd/f/05113e00edc94deeb1d57cf4e2bde358 for snapshot=snapshot_1700357198794_ns2_test-17003571846791 2023-11-19T01:26:39,862 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd. 2023-11-19T01:26:39,862 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=35 2023-11-19T01:26:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=35 2023-11-19T01:26:39,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357198794_ns2_test-17003571846791 on region dd0bac2484821b3c9f7478186b974ffd 2023-11-19T01:26:39,864 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=35, ppid=34, state=RUNNABLE; SnapshotRegionProcedure dd0bac2484821b3c9f7478186b974ffd 2023-11-19T01:26:39,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=35, resume processing ppid=34 2023-11-19T01:26:39,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=35, ppid=34, state=SUCCESS; SnapshotRegionProcedure dd0bac2484821b3c9f7478186b974ffd in 1.0370 sec 2023-11-19T01:26:39,868 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357198794_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:26:39,869 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357198794_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:26:39,870 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357198794_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:26:39,870 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357198794_ns2_test-17003571846791 2023-11-19T01:26:39,871 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357198794_ns2_test-17003571846791 2023-11-19T01:26:39,899 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357198794_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:26:39,907 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357198794_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:26:39,908 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357198794_ns2_test-17003571846791 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357198794_ns2_test-17003571846791 2023-11-19T01:26:39,909 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=34, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357198794_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:26:39,909 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357198794_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 }, snapshot procedure id = 34 2023-11-19T01:26:39,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=34, snapshot={ ss=snapshot_1700357198794_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } in 1.1010 sec 2023-11-19T01:26:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=34 2023-11-19T01:26:39,917 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: ns2:test-17003571846791, procId: 34 completed 2023-11-19T01:26:39,918 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357199917_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } 2023-11-19T01:26:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357199918 (current time:1700357199918). 2023-11-19T01:26:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:26:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357199917_ns3_test-17003571846792 VERSION not specified, setting to 2 2023-11-19T01:26:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x511565ce to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:26:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5331d9b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:26:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:26:39,930 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:39864, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:26:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x511565ce to 127.0.0.1:57501 2023-11-19T01:26:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:26:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:26:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=36, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1700357199917_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } 2023-11-19T01:26:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357199917_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-11-19T01:26:39,936 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1700357199917_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:26:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-19T01:26:39,938 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1700357199917_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:26:39,942 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357199917_ns3_test-17003571846792 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:26:39,945 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1700357199917_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:26:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-19T01:26:40,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-19T01:26:40,365 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1700357199917_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:26:40,365 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 628b78cfbf31117825b170549256b565}] 2023-11-19T01:26:40,367 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 628b78cfbf31117825b170549256b565 2023-11-19T01:26:40,519 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:40,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=37 2023-11-19T01:26:40,520 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565. 2023-11-19T01:26:40,521 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.HRegion(2520): Flush status journal for 628b78cfbf31117825b170549256b565: 2023-11-19T01:26:40,521 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565. for snapshot_1700357199917_ns3_test-17003571846792 completed. 2023-11-19T01:26:40,521 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(240): Storing 'ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565.' region-info for snapshot=snapshot_1700357199917_ns3_test-17003571846792 2023-11-19T01:26:40,522 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:26:40,522 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-19T01:26:40,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-19T01:26:40,932 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565. 2023-11-19T01:26:40,932 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=37 2023-11-19T01:26:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=37 2023-11-19T01:26:40,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357199917_ns3_test-17003571846792 on region 628b78cfbf31117825b170549256b565 2023-11-19T01:26:40,934 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=37, ppid=36, state=RUNNABLE; SnapshotRegionProcedure 628b78cfbf31117825b170549256b565 2023-11-19T01:26:40,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=37, resume processing ppid=36 2023-11-19T01:26:40,937 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1700357199917_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:26:40,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=37, ppid=36, state=SUCCESS; SnapshotRegionProcedure 628b78cfbf31117825b170549256b565 in 570 msec 2023-11-19T01:26:40,938 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1700357199917_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:26:40,939 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1700357199917_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:26:40,939 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357199917_ns3_test-17003571846792 2023-11-19T01:26:40,940 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357199917_ns3_test-17003571846792 2023-11-19T01:26:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-19T01:26:41,354 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1700357199917_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:26:41,360 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1700357199917_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:26:41,361 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357199917_ns3_test-17003571846792 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357199917_ns3_test-17003571846792 2023-11-19T01:26:41,362 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=36, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1700357199917_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:26:41,362 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357199917_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 }, snapshot procedure id = 36 2023-11-19T01:26:41,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=36, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=36, snapshot={ ss=snapshot_1700357199917_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } in 1.4300 sec 2023-11-19T01:26:42,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=36 2023-11-19T01:26:42,045 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: ns3:test-17003571846792, procId: 36 completed 2023-11-19T01:26:42,045 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1700357192923 2023-11-19T01:26:42,045 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-19T01:26:42,047 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-11-19T01:26:42,049 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357192923_ns2:test-17003571846791 2023-11-19T01:26:42,049 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357198794_ns2_test-17003571846791 to hdfs://localhost:42739/backupUT/backup_1700357192923/ns2/test-17003571846791/ 2023-11-19T01:26:42,060 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:26:42,093 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:26:42,093 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,44889,1700357173572_48372649_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42739/backupUT/backup_1700357192923/ns2/test-17003571846791, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42739/backupUT/backup_1700357192923/ns2/test-17003571846791/.hbase-snapshot/.tmp/snapshot_1700357198794_ns2_test-17003571846791 2023-11-19T01:26:42,093 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af, snapshotDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357198794_ns2_test-17003571846791. 2023-11-19T01:26:42,106 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357198794_ns2_test-17003571846791 to hdfs://localhost:42739/backupUT/backup_1700357192923/ns2/test-17003571846791/.hbase-snapshot/.tmp/snapshot_1700357198794_ns2_test-17003571846791 2023-11-19T01:26:42,140 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:42,141 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:42,141 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:42,142 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:43,216 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-3896633903243007058.jar 2023-11-19T01:26:43,217 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:43,217 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:43,218 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:43,218 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:43,219 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:43,219 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:43,219 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:43,220 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:43,220 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:43,221 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:26:43,222 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:26:43,222 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:26:43,223 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:26:43,223 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:26:43,224 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:26:43,224 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:26:43,225 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:26:43,225 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:26:43,226 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:26:43,226 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:26:43,227 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:26:43,231 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:26:43,231 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:26:43,232 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:26:43,232 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:26:43,232 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:26:43,233 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:26:43,234 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:26:43,294 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-19T01:26:43,320 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357198794_ns2_test-17003571846791' hfile list 2023-11-19T01:26:43,334 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-11-19T01:26:43,553 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357203504/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:26:43,553 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357203504/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:26:43,718 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-19T01:26:43,748 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd/f/05113e00edc94deeb1d57cf4e2bde358 output=hdfs://localhost:42739/backupUT/backup_1700357192923/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd/f/05113e00edc94deeb1d57cf4e2bde358 2023-11-19T01:26:43,748 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-11-19T01:26:44,613 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:26:44,614 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:26:44,621 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357198794_ns2_test-17003571846791 2023-11-19T01:26:44,621 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357198794_ns2_test-17003571846791 finished. 2023-11-19T01:26:44,621 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357192923_ns3:test-17003571846792 2023-11-19T01:26:44,622 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357199917_ns3_test-17003571846792 to hdfs://localhost:42739/backupUT/backup_1700357192923/ns3/test-17003571846792/ 2023-11-19T01:26:44,622 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:26:44,647 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:26:44,648 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,44889,1700357173572_-1973369624_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42739/backupUT/backup_1700357192923/ns3/test-17003571846792, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42739/backupUT/backup_1700357192923/ns3/test-17003571846792/.hbase-snapshot/.tmp/snapshot_1700357199917_ns3_test-17003571846792 2023-11-19T01:26:44,648 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af, snapshotDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357199917_ns3_test-17003571846792. 2023-11-19T01:26:44,654 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357199917_ns3_test-17003571846792 to hdfs://localhost:42739/backupUT/backup_1700357192923/ns3/test-17003571846792/.hbase-snapshot/.tmp/snapshot_1700357199917_ns3_test-17003571846792 2023-11-19T01:26:44,784 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk 2023-11-19T01:26:44,784 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_backup_table_system_bulk Metrics about Tables on a single HBase RegionServer 2023-11-19T01:26:45,083 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:45,083 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:45,084 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:45,084 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:46,109 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-5055222411152839491.jar 2023-11-19T01:26:46,110 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:46,110 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:46,111 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:46,111 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:46,112 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:46,112 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:46,113 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:46,113 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:46,114 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:46,114 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:26:46,115 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:26:46,115 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:26:46,116 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:26:46,117 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:26:46,117 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:26:46,118 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:26:46,118 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:26:46,119 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:26:46,120 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:26:46,121 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:26:46,121 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:26:46,122 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:26:46,123 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:26:46,124 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:26:46,125 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:26:46,125 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:26:46,126 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:26:46,127 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:26:46,157 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-19T01:26:46,176 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357199917_ns3_test-17003571846792' hfile list 2023-11-19T01:26:46,261 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357206225/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:26:46,261 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357206225/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:26:47,306 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:26:47,309 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:26:47,315 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357199917_ns3_test-17003571846792 2023-11-19T01:26:47,315 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357199917_ns3_test-17003571846792 finished. 2023-11-19T01:26:47,316 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1034): ns2:test-17003571846791 2023-11-19T01:26:47,316 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1034): ns3:test-17003571846792 2023-11-19T01:26:47,393 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357192923 2023-11-19T01:26:47,393 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:26:47,805 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42739/backupUT/backup_1700357192923/.backup.manifest 2023-11-19T01:26:47,805 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357192923 2023-11-19T01:26:47,805 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:26:48,211 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42739/backupUT/backup_1700357192923/.backup.manifest 2023-11-19T01:26:48,212 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(358): Backup backup_1700357192923 finished: type=FULL,tablelist=ns2:test-17003571846791;ns3:test-17003571846792,targetRootDir=hdfs://localhost:42739/backupUT,startts=1700357198505,completets=1700357207391,bytescopied=0 2023-11-19T01:26:48,212 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-19T01:26:48,212 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357198794_ns2_test-17003571846791 2023-11-19T01:26:48,222 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357198794_ns2_test-17003571846791" 2023-11-19T01:26:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357198794_ns2_test-17003571846791 2023-11-19T01:26:48,227 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357198794_ns2_test-17003571846791 for backup backup_1700357192923 succeeded. 2023-11-19T01:26:48,227 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357199917_ns3_test-17003571846792 2023-11-19T01:26:48,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357199917_ns3_test-17003571846792" 2023-11-19T01:26:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357199917_ns3_test-17003571846792 2023-11-19T01:26:48,232 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357199917_ns3_test-17003571846792 for backup backup_1700357192923 succeeded. 2023-11-19T01:26:48,232 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:26:48,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:26:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:26:48,248 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:26:48,255 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:26:48,277 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(376): Backup backup_1700357192923 completed. 2023-11-19T01:26:48,277 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:26:48,277 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x11ad338b to 127.0.0.1:57501 2023-11-19T01:26:48,277 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:26:48,305 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357192923/.backup.manifest 2023-11-19T01:26:48,307 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357192923/.backup.manifest 2023-11-19T01:26:48,308 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357192923/.backup.manifest 2023-11-19T01:26:48,310 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357192923/.backup.manifest 2023-11-19T01:26:48,316 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357192923/.backup.manifest 2023-11-19T01:26:48,318 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357192923/.backup.manifest 2023-11-19T01:26:48,318 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-17003571846791' to 'ns2:test-17003571846791' from full backup image hdfs://localhost:42739/backupUT/backup_1700357192923/ns2/test-17003571846791 2023-11-19T01:26:48,330 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(476): Truncating exising target table 'ns2:test-17003571846791', preserving region splits 2023-11-19T01:26:48,332 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$18(967): Started disable of ns2:test-17003571846791 2023-11-19T01:26:48,338 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable ns2:test-17003571846791 2023-11-19T01:26:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=38, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:test-17003571846791 2023-11-19T01:26:48,351 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-17003571846791","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357208351"}]},"ts":"1700357208351"} 2023-11-19T01:26:48,353 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-17003571846791, state=DISABLING in hbase:meta 2023-11-19T01:26:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-19T01:26:48,356 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set ns2:test-17003571846791 to state=DISABLING 2023-11-19T01:26:48,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-17003571846791, region=dd0bac2484821b3c9f7478186b974ffd, UNASSIGN}] 2023-11-19T01:26:48,360 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-17003571846791, region=dd0bac2484821b3c9f7478186b974ffd, UNASSIGN 2023-11-19T01:26:48,361 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=dd0bac2484821b3c9f7478186b974ffd, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:48,363 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE; CloseRegionProcedure dd0bac2484821b3c9f7478186b974ffd, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:26:48,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-19T01:26:48,524 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:48,526 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(119): Close dd0bac2484821b3c9f7478186b974ffd 2023-11-19T01:26:48,526 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1668): Closing dd0bac2484821b3c9f7478186b974ffd, disabling compactions & flushes 2023-11-19T01:26:48,526 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1690): Closing region ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd. 2023-11-19T01:26:48,526 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd. 2023-11-19T01:26:48,526 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1778): Acquired close lock on ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd. after waiting 0 ms 2023-11-19T01:26:48,526 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1788): Updates disabled for region ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd. 2023-11-19T01:26:48,536 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-11-19T01:26:48,537 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:26:48,538 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1909): Closed ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd. 2023-11-19T01:26:48,539 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1622): Region close journal for dd0bac2484821b3c9f7478186b974ffd: 2023-11-19T01:26:48,541 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(163): Closed dd0bac2484821b3c9f7478186b974ffd 2023-11-19T01:26:48,542 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=39 updating hbase:meta row=dd0bac2484821b3c9f7478186b974ffd, regionState=CLOSED 2023-11-19T01:26:48,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=40, resume processing ppid=39 2023-11-19T01:26:48,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=40, ppid=39, state=SUCCESS; CloseRegionProcedure dd0bac2484821b3c9f7478186b974ffd, server=jenkins-hbase6.apache.org,44997,1700357176784 in 181 msec 2023-11-19T01:26:48,551 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=39, resume processing ppid=38 2023-11-19T01:26:48,551 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=39, ppid=38, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-17003571846791, region=dd0bac2484821b3c9f7478186b974ffd, UNASSIGN in 190 msec 2023-11-19T01:26:48,552 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-17003571846791","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357208552"}]},"ts":"1700357208552"} 2023-11-19T01:26:48,554 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-17003571846791, state=DISABLED in hbase:meta 2023-11-19T01:26:48,556 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set ns2:test-17003571846791 to state=DISABLED 2023-11-19T01:26:48,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=38, state=SUCCESS; DisableTableProcedure table=ns2:test-17003571846791 in 217 msec 2023-11-19T01:26:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=38 2023-11-19T01:26:48,659 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$TableFuture(3745): Operation: DISABLE, Table Name: ns2:test-17003571846791, procId: 38 completed 2023-11-19T01:26:48,661 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$16(847): Started truncating ns2:test-17003571846791 2023-11-19T01:26:48,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$6(2481): Client=jenkins//172.31.2.20 truncate ns2:test-17003571846791 2023-11-19T01:26:48,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=41, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns2:test-17003571846791 preserveSplits=true) 2023-11-19T01:26:48,678 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(87): waiting for 'ns2:test-17003571846791' regions in transition 2023-11-19T01:26:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-19T01:26:48,686 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd 2023-11-19T01:26:48,690 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd/f, FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd/recovered.edits] 2023-11-19T01:26:48,698 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd/f/05113e00edc94deeb1d57cf4e2bde358 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/archive/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd/f/05113e00edc94deeb1d57cf4e2bde358 2023-11-19T01:26:48,701 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd/recovered.edits/106.seqid to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/archive/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd/recovered.edits/106.seqid 2023-11-19T01:26:48,702 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd 2023-11-19T01:26:48,702 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived ns2:test-17003571846791 regions 2023-11-19T01:26:48,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2023-11-19T01:26:48,715 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:test-17003571846791 from hbase:meta 2023-11-19T01:26:48,717 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:test-17003571846791' descriptor. 2023-11-19T01:26:48,718 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:test-17003571846791' from region states. 2023-11-19T01:26:48,718 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357208718"}]},"ts":"9223372036854775807"} 2023-11-19T01:26:48,720 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:26:48,720 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => dd0bac2484821b3c9f7478186b974ffd, NAME => 'ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:26:48,720 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:test-17003571846791' as deleted. 2023-11-19T01:26:48,721 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-17003571846791","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1700357208720"}]},"ts":"9223372036854775807"} 2023-11-19T01:26:48,723 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:test-17003571846791 state from META 2023-11-19T01:26:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-19T01:26:48,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-19T01:26:49,134 INFO [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 4ef412d65669b2fe1684b7fdf6ee1f7d, NAME => 'ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-17003571846791', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:26:49,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-19T01:26:49,542 DEBUG [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:49,542 DEBUG [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1668): Closing 4ef412d65669b2fe1684b7fdf6ee1f7d, disabling compactions & flushes 2023-11-19T01:26:49,542 INFO [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. 2023-11-19T01:26:49,542 DEBUG [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. 2023-11-19T01:26:49,542 DEBUG [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. after waiting 0 ms 2023-11-19T01:26:49,542 DEBUG [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. 2023-11-19T01:26:49,542 INFO [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. 2023-11-19T01:26:49,542 DEBUG [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1622): Region close journal for 4ef412d65669b2fe1684b7fdf6ee1f7d: 2023-11-19T01:26:49,544 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1700357209543"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357209543"}]},"ts":"1700357209543"} 2023-11-19T01:26:49,546 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:26:49,547 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-17003571846791","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357209546"}]},"ts":"1700357209546"} 2023-11-19T01:26:49,548 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-17003571846791, state=ENABLING in hbase:meta 2023-11-19T01:26:49,553 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-17003571846791, region=4ef412d65669b2fe1684b7fdf6ee1f7d, ASSIGN}] 2023-11-19T01:26:49,555 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-17003571846791, region=4ef412d65669b2fe1684b7fdf6ee1f7d, ASSIGN 2023-11-19T01:26:49,556 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-17003571846791, region=4ef412d65669b2fe1684b7fdf6ee1f7d, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,44997,1700357176784; forceNewPlan=false, retain=false 2023-11-19T01:26:49,707 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=42 updating hbase:meta row=4ef412d65669b2fe1684b7fdf6ee1f7d, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:49,709 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; OpenRegionProcedure 4ef412d65669b2fe1684b7fdf6ee1f7d, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:26:49,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-19T01:26:49,861 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:49,866 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(130): Open ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. 2023-11-19T01:26:49,866 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7254): Opening region: {ENCODED => 4ef412d65669b2fe1684b7fdf6ee1f7d, NAME => 'ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:26:49,866 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:26:49,867 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17003571846791 4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:26:49,867 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(888): Instantiated ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:49,867 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7296): checking encryption for 4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:26:49,867 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7299): checking classloading for 4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:26:49,868 INFO [StoreOpener-4ef412d65669b2fe1684b7fdf6ee1f7d-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:26:49,870 INFO [StoreOpener-4ef412d65669b2fe1684b7fdf6ee1f7d-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4ef412d65669b2fe1684b7fdf6ee1f7d columnFamilyName f 2023-11-19T01:26:49,870 DEBUG [StoreOpener-4ef412d65669b2fe1684b7fdf6ee1f7d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:49,871 INFO [StoreOpener-4ef412d65669b2fe1684b7fdf6ee1f7d-1 {}] regionserver.HStore(324): Store=4ef412d65669b2fe1684b7fdf6ee1f7d/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:26:49,872 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:26:49,872 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:26:49,875 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1076): writing seq id for 4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:26:49,878 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:26:49,878 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): Opened 4ef412d65669b2fe1684b7fdf6ee1f7d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10853043360, jitterRate=0.010768428444862366}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:26:49,879 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(992): Region open journal for 4ef412d65669b2fe1684b7fdf6ee1f7d: 2023-11-19T01:26:49,880 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d., pid=43, masterSystemTime=1700357209861 2023-11-19T01:26:49,882 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. 2023-11-19T01:26:49,882 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(158): Opened ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. 2023-11-19T01:26:49,882 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=42 updating hbase:meta row=4ef412d65669b2fe1684b7fdf6ee1f7d, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:49,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=43, resume processing ppid=42 2023-11-19T01:26:49,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=43, ppid=42, state=SUCCESS; OpenRegionProcedure 4ef412d65669b2fe1684b7fdf6ee1f7d, server=jenkins-hbase6.apache.org,44997,1700357176784 in 175 msec 2023-11-19T01:26:49,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=42, resume processing ppid=41 2023-11-19T01:26:49,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=42, ppid=41, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-17003571846791, region=4ef412d65669b2fe1684b7fdf6ee1f7d, ASSIGN in 333 msec 2023-11-19T01:26:49,888 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-17003571846791","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357209888"}]},"ts":"1700357209888"} 2023-11-19T01:26:49,890 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-17003571846791, state=ENABLED in hbase:meta 2023-11-19T01:26:49,897 DEBUG [PEWorker-3 {}] procedure.TruncateTableProcedure(145): truncate 'ns2:test-17003571846791' completed 2023-11-19T01:26:49,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=41, state=SUCCESS; TruncateTableProcedure (table=ns2:test-17003571846791 preserveSplits=true) in 1.2280 sec 2023-11-19T01:26:50,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=41 2023-11-19T01:26:50,786 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$TableFuture(3745): Operation: TRUNCATE, Table Name: ns2:test-17003571846791, procId: 41 completed 2023-11-19T01:26:50,793 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:42739/backupUT/backup_1700357192923/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd from hbase tables ns2:test-17003571846791 to tables ns2:test-17003571846791 2023-11-19T01:26:50,793 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-17003571846791 into ns2:test-17003571846791 2023-11-19T01:26:50,796 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-test-17003571846791-1700357210794 from hdfs://localhost:42739/backupUT/backup_1700357192923/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd 2023-11-19T01:26:50,800 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x238cf690 to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:26:50,805 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@696c9b4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:26:50,807 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:26:50,808 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43160, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:26:50,822 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-19T01:26:50,823 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-17003571846791 2023-11-19T01:26:50,827 DEBUG [hconnection-0x18a23597-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:26:50,828 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:50484, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:26:50,831 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-17003571846791: 2023-11-19T01:26:50,832 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-11-19T01:26:50,832 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/partitions_24f19fcf-8651-4714-bb59-8d88160d8942 2023-11-19T01:26:51,111 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2023-11-19T01:26:51,113 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43174, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2023-11-19T01:26:51,279 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:51,279 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:51,280 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:51,280 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:52,286 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-8802143916775570511.jar 2023-11-19T01:26:52,286 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:52,287 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:52,287 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:52,288 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:52,288 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:52,289 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:52,289 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:52,290 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:52,290 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:52,291 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:26:52,292 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:26:52,292 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:26:52,293 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:26:52,294 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:26:52,294 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:26:52,295 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:26:52,296 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:26:52,296 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:26:52,297 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:26:52,298 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:26:52,298 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:26:52,302 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:52,303 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:52,304 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:52,304 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:52,305 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:52,305 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:26:52,306 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:26:52,307 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-17003571846791 2023-11-19T01:26:52,307 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:26:52,307 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x238cf690 to 127.0.0.1:57501 2023-11-19T01:26:52,307 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:26:52,307 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-11-19T01:26:52,307 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-11-19T01:26:52,308 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:26:52,315 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-19T01:26:52,321 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-11-19T01:26:52,413 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357212376/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:26:52,413 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357212376/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:26:52,494 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:42739/backupUT/backup_1700357192923/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/dd0bac2484821b3c9f7478186b974ffd/f/05113e00edc94deeb1d57cf4e2bde358 2023-11-19T01:26:52,502 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-11-19T01:26:52,660 DEBUG [pool-203-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3ae521cc to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:26:52,668 DEBUG [pool-203-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dc6e180, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:26:52,670 DEBUG [hconnection-0x397afd33-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:26:52,672 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:50496, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:26:52,678 DEBUG [pool-203-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3ae521cc to 127.0.0.1:57501 2023-11-19T01:26:52,678 DEBUG [pool-203-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:26:52,678 DEBUG [pool-203-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-11-19T01:26:52,678 DEBUG [pool-203-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase6.apache.org 2023-11-19T01:26:52,700 DEBUG [pool-203-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-test-17003571846791-1700357210794/_temporary/0/_temporary/attempt_local1848443168_0003_r_000000_0/f/1dde304b0adf4e47839adf1b909fe66e is 37, key is row10/f:q1/1700357188531/Put/seqid=0 2023-11-19T01:26:52,703 WARN [Thread-797 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase6.apache.org:44997] Specified favored nodes: [jenkins-hbase6.apache.org:44997] 2023-11-19T01:26:53,481 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-test-17003571846791-1700357210794 2023-11-19T01:26:53,483 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x49e194fe to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:26:53,491 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33fb7e73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:26:53,493 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:26:53,494 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:50500, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:26:53,504 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-test-17003571846791-1700357210794/_SUCCESS 2023-11-19T01:26:53,510 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:26:53,512 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43178, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:26:53,587 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-test-17003571846791-1700357210794/f/1dde304b0adf4e47839adf1b909fe66e first=Optional[row0] last=Optional[row98] 2023-11-19T01:26:53,615 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d., hostname=jenkins-hbase6.apache.org,44997,1700357176784, seqNum=2 for row with hfile group [{f,hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-test-17003571846791-1700357210794/f/1dde304b0adf4e47839adf1b909fe66e}] 2023-11-19T01:26:53,643 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-test-17003571846791-1700357210794/f/1dde304b0adf4e47839adf1b909fe66e for inclusion in 4ef412d65669b2fe1684b7fdf6ee1f7d/f 2023-11-19T01:26:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-11-19T01:26:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(625): Region bounds: first= last= 2023-11-19T01:26:53,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegion(2520): Flush status journal for 4ef412d65669b2fe1684b7fdf6ee1f7d: 2023-11-19T01:26:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-test-17003571846791-1700357210794/f/1dde304b0adf4e47839adf1b909fe66e to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__ns2_test-17003571846791__j6ojnhj5jsssckunivg51l7v8furopu6gk37kcj2oru0mriuhknv34i7ua99buk5/f/1dde304b0adf4e47839adf1b909fe66e 2023-11-19T01:26:53,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__ns2_test-17003571846791__j6ojnhj5jsssckunivg51l7v8furopu6gk37kcj2oru0mriuhknv34i7ua99buk5/f/1dde304b0adf4e47839adf1b909fe66e as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_ 2023-11-19T01:26:53,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x08ce55cd to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:26:53,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d147fb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:26:53,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:26:53,674 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43190, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:26:53,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:26:53,681 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:50514, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:26:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(428): write bulk load descriptor to backup ns2:test-17003571846791 with 1 entries 2023-11-19T01:26:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(1602): writing raw bulk path hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_ for ns2:test-17003571846791 4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:26:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(435): written 1 rows for bulk load of ns2:test-17003571846791 2023-11-19T01:26:53,714 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:26:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x08ce55cd to 127.0.0.1:57501 2023-11-19T01:26:53,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:26:53,716 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__ns2_test-17003571846791__j6ojnhj5jsssckunivg51l7v8furopu6gk37kcj2oru0mriuhknv34i7ua99buk5/f/1dde304b0adf4e47839adf1b909fe66e into 4ef412d65669b2fe1684b7fdf6ee1f7d/f as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_ - updating store file list. 2023-11-19T01:26:53,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_ into 4ef412d65669b2fe1684b7fdf6ee1f7d/f 2023-11-19T01:26:53,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__ns2_test-17003571846791__j6ojnhj5jsssckunivg51l7v8furopu6gk37kcj2oru0mriuhknv34i7ua99buk5/f/1dde304b0adf4e47839adf1b909fe66e into 4ef412d65669b2fe1684b7fdf6ee1f7d/f (new location: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_) 2023-11-19T01:26:53,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__ns2_test-17003571846791__j6ojnhj5jsssckunivg51l7v8furopu6gk37kcj2oru0mriuhknv34i7ua99buk5/f/1dde304b0adf4e47839adf1b909fe66e 2023-11-19T01:26:53,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4336840c to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:26:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc88ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:26:53,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:26:53,748 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43204, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-19T01:26:53,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:26:53,754 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:50522, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-11-19T01:26:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup ns2:test-17003571846791 with 1 entries 2023-11-19T01:26:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(1523): writing done bulk path hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_ for ns2:test-17003571846791 4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:26:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(414): written 1 rows for bulk load of ns2:test-17003571846791 2023-11-19T01:26:53,777 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:26:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4336840c to 127.0.0.1:57501 2023-11-19T01:26:53,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:26:53,789 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:26:53,789 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x49e194fe to 127.0.0.1:57501 2023-11-19T01:26:53,790 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:26:53,790 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-11-19T01:26:53,790 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-19T01:26:53,790 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(233): Backup: backup_1700357192923 hdfs://localhost:42739/backupUT/backup_1700357192923/ns2/test-17003571846791/ 2023-11-19T01:26:53,792 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357192923/.backup.manifest 2023-11-19T01:26:53,796 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357192923/.backup.manifest 2023-11-19T01:26:53,796 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-17003571846792' to 'ns3:test-17003571846792' from full backup image hdfs://localhost:42739/backupUT/backup_1700357192923/ns3/test-17003571846792 2023-11-19T01:26:53,805 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:42739/backupUT/backup_1700357192923/ns3/test-17003571846792/archive/data/ns3/test-17003571846792 does not exists 2023-11-19T01:26:53,805 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-17003571846792, will only create table 2023-11-19T01:26:53,807 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(476): Truncating exising target table 'ns3:test-17003571846792', preserving region splits 2023-11-19T01:26:53,808 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$18(967): Started disable of ns3:test-17003571846792 2023-11-19T01:26:53,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable ns3:test-17003571846792 2023-11-19T01:26:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=44, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:test-17003571846792 2023-11-19T01:26:53,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2023-11-19T01:26:53,813 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-17003571846792","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357213813"}]},"ts":"1700357213813"} 2023-11-19T01:26:53,815 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-17003571846792, state=DISABLING in hbase:meta 2023-11-19T01:26:53,818 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set ns3:test-17003571846792 to state=DISABLING 2023-11-19T01:26:53,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-17003571846792, region=628b78cfbf31117825b170549256b565, UNASSIGN}] 2023-11-19T01:26:53,821 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=45, ppid=44, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-17003571846792, region=628b78cfbf31117825b170549256b565, UNASSIGN 2023-11-19T01:26:53,822 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=628b78cfbf31117825b170549256b565, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:53,824 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; CloseRegionProcedure 628b78cfbf31117825b170549256b565, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:26:53,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2023-11-19T01:26:53,977 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:53,977 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(119): Close 628b78cfbf31117825b170549256b565 2023-11-19T01:26:53,978 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1668): Closing 628b78cfbf31117825b170549256b565, disabling compactions & flushes 2023-11-19T01:26:53,978 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1690): Closing region ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565. 2023-11-19T01:26:53,978 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565. 2023-11-19T01:26:53,978 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1778): Acquired close lock on ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565. after waiting 0 ms 2023-11-19T01:26:53,978 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1788): Updates disabled for region ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565. 2023-11-19T01:26:53,984 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/628b78cfbf31117825b170549256b565/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2023-11-19T01:26:53,985 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:26:53,986 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1909): Closed ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565. 2023-11-19T01:26:53,986 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] regionserver.HRegion(1622): Region close journal for 628b78cfbf31117825b170549256b565: 2023-11-19T01:26:53,988 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=46}] handler.UnassignRegionHandler(163): Closed 628b78cfbf31117825b170549256b565 2023-11-19T01:26:53,989 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=45 updating hbase:meta row=628b78cfbf31117825b170549256b565, regionState=CLOSED 2023-11-19T01:26:53,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=46, resume processing ppid=45 2023-11-19T01:26:53,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=46, ppid=45, state=SUCCESS; CloseRegionProcedure 628b78cfbf31117825b170549256b565, server=jenkins-hbase6.apache.org,44997,1700357176784 in 166 msec 2023-11-19T01:26:53,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=45, resume processing ppid=44 2023-11-19T01:26:53,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=45, ppid=44, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-17003571846792, region=628b78cfbf31117825b170549256b565, UNASSIGN in 174 msec 2023-11-19T01:26:53,996 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-17003571846792","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357213995"}]},"ts":"1700357213995"} 2023-11-19T01:26:53,997 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-17003571846792, state=DISABLED in hbase:meta 2023-11-19T01:26:54,000 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set ns3:test-17003571846792 to state=DISABLED 2023-11-19T01:26:54,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=44, state=SUCCESS; DisableTableProcedure table=ns3:test-17003571846792 in 192 msec 2023-11-19T01:26:54,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=44 2023-11-19T01:26:54,115 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$TableFuture(3745): Operation: DISABLE, Table Name: ns3:test-17003571846792, procId: 44 completed 2023-11-19T01:26:54,116 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$16(847): Started truncating ns3:test-17003571846792 2023-11-19T01:26:54,116 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$6(2481): Client=jenkins//172.31.2.20 truncate ns3:test-17003571846792 2023-11-19T01:26:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=47, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns3:test-17003571846792 preserveSplits=true) 2023-11-19T01:26:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-19T01:26:54,119 DEBUG [PEWorker-4 {}] procedure.TruncateTableProcedure(87): waiting for 'ns3:test-17003571846792' regions in transition 2023-11-19T01:26:54,123 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/628b78cfbf31117825b170549256b565 2023-11-19T01:26:54,124 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/628b78cfbf31117825b170549256b565/f, FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/628b78cfbf31117825b170549256b565/recovered.edits] 2023-11-19T01:26:54,128 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:26:54,138 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/628b78cfbf31117825b170549256b565/recovered.edits/5.seqid to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/archive/data/ns3/test-17003571846792/628b78cfbf31117825b170549256b565/recovered.edits/5.seqid 2023-11-19T01:26:54,138 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/628b78cfbf31117825b170549256b565 2023-11-19T01:26:54,139 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(293): Archived ns3:test-17003571846792 regions 2023-11-19T01:26:54,148 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:test-17003571846792 from hbase:meta 2023-11-19T01:26:54,152 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:test-17003571846792' descriptor. 2023-11-19T01:26:54,153 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:test-17003571846792' from region states. 2023-11-19T01:26:54,153 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357214153"}]},"ts":"9223372036854775807"} 2023-11-19T01:26:54,156 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:26:54,156 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 628b78cfbf31117825b170549256b565, NAME => 'ns3:test-17003571846792,,1700357188625.628b78cfbf31117825b170549256b565.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:26:54,156 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:test-17003571846792' as deleted. 2023-11-19T01:26:54,156 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-17003571846792","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1700357214156"}]},"ts":"9223372036854775807"} 2023-11-19T01:26:54,159 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:test-17003571846792 state from META 2023-11-19T01:26:54,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-19T01:26:54,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-19T01:26:54,578 INFO [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => b78f59bec877d4d278f3912d6fb4631e, NAME => 'ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-17003571846792', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:26:54,588 DEBUG [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:54,588 DEBUG [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1668): Closing b78f59bec877d4d278f3912d6fb4631e, disabling compactions & flushes 2023-11-19T01:26:54,589 INFO [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. 2023-11-19T01:26:54,589 DEBUG [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. 2023-11-19T01:26:54,589 DEBUG [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. after waiting 0 ms 2023-11-19T01:26:54,589 DEBUG [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. 2023-11-19T01:26:54,589 INFO [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. 2023-11-19T01:26:54,589 DEBUG [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1622): Region close journal for b78f59bec877d4d278f3912d6fb4631e: 2023-11-19T01:26:54,590 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1700357214590"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357214590"}]},"ts":"1700357214590"} 2023-11-19T01:26:54,592 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:26:54,594 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-17003571846792","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357214593"}]},"ts":"1700357214593"} 2023-11-19T01:26:54,595 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-17003571846792, state=ENABLING in hbase:meta 2023-11-19T01:26:54,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-17003571846792, region=b78f59bec877d4d278f3912d6fb4631e, ASSIGN}] 2023-11-19T01:26:54,603 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-17003571846792, region=b78f59bec877d4d278f3912d6fb4631e, ASSIGN 2023-11-19T01:26:54,604 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-17003571846792, region=b78f59bec877d4d278f3912d6fb4631e, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,44997,1700357176784; forceNewPlan=false, retain=false 2023-11-19T01:26:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-19T01:26:54,756 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=b78f59bec877d4d278f3912d6fb4631e, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:54,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE; OpenRegionProcedure b78f59bec877d4d278f3912d6fb4631e, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:26:54,784 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17003571846792 2023-11-19T01:26:54,913 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:54,917 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(130): Open ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. 2023-11-19T01:26:54,917 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7254): Opening region: {ENCODED => b78f59bec877d4d278f3912d6fb4631e, NAME => 'ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:26:54,918 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:26:54,918 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17003571846792 b78f59bec877d4d278f3912d6fb4631e 2023-11-19T01:26:54,918 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(888): Instantiated ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:26:54,918 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7296): checking encryption for b78f59bec877d4d278f3912d6fb4631e 2023-11-19T01:26:54,918 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7299): checking classloading for b78f59bec877d4d278f3912d6fb4631e 2023-11-19T01:26:54,919 INFO [StoreOpener-b78f59bec877d4d278f3912d6fb4631e-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region b78f59bec877d4d278f3912d6fb4631e 2023-11-19T01:26:54,921 INFO [StoreOpener-b78f59bec877d4d278f3912d6fb4631e-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b78f59bec877d4d278f3912d6fb4631e columnFamilyName f 2023-11-19T01:26:54,921 DEBUG [StoreOpener-b78f59bec877d4d278f3912d6fb4631e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:26:54,922 INFO [StoreOpener-b78f59bec877d4d278f3912d6fb4631e-1 {}] regionserver.HStore(324): Store=b78f59bec877d4d278f3912d6fb4631e/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:26:54,922 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/b78f59bec877d4d278f3912d6fb4631e 2023-11-19T01:26:54,923 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/b78f59bec877d4d278f3912d6fb4631e 2023-11-19T01:26:54,925 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1076): writing seq id for b78f59bec877d4d278f3912d6fb4631e 2023-11-19T01:26:54,927 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/b78f59bec877d4d278f3912d6fb4631e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:26:54,928 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): Opened b78f59bec877d4d278f3912d6fb4631e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9886432960, jitterRate=-0.07925418019294739}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:26:54,929 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(992): Region open journal for b78f59bec877d4d278f3912d6fb4631e: 2023-11-19T01:26:54,929 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e., pid=49, masterSystemTime=1700357214913 2023-11-19T01:26:54,931 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. 2023-11-19T01:26:54,931 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(158): Opened ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. 2023-11-19T01:26:54,932 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=48 updating hbase:meta row=b78f59bec877d4d278f3912d6fb4631e, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:54,936 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=49, resume processing ppid=48 2023-11-19T01:26:54,936 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=49, ppid=48, state=SUCCESS; OpenRegionProcedure b78f59bec877d4d278f3912d6fb4631e, server=jenkins-hbase6.apache.org,44997,1700357176784 in 175 msec 2023-11-19T01:26:54,938 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=48, resume processing ppid=47 2023-11-19T01:26:54,938 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=48, ppid=47, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-17003571846792, region=b78f59bec877d4d278f3912d6fb4631e, ASSIGN in 335 msec 2023-11-19T01:26:54,938 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-17003571846792","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357214938"}]},"ts":"1700357214938"} 2023-11-19T01:26:54,939 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-17003571846792, state=ENABLED in hbase:meta 2023-11-19T01:26:54,941 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(145): truncate 'ns3:test-17003571846792' completed 2023-11-19T01:26:54,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=47, state=SUCCESS; TruncateTableProcedure (table=ns3:test-17003571846792 preserveSplits=true) in 825 msec 2023-11-19T01:26:55,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=47 2023-11-19T01:26:55,226 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$TableFuture(3745): Operation: TRUNCATE, Table Name: ns3:test-17003571846792, procId: 47 completed 2023-11-19T01:26:55,229 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-19T01:26:55,230 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(233): Backup: backup_1700357192923 hdfs://localhost:42739/backupUT/backup_1700357192923/ns3/test-17003571846792/ 2023-11-19T01:26:55,230 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-11-19T01:26:55,251 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleOverwrite Thread=600 (was 584) Potentially hanging thread: HFileArchiver-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-54279523_16 at /127.0.0.1:57792 [Receiving block BP-935274450-172.31.2.20-1700357170119:blk_1073741856_1032] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-54279523_16 at /127.0.0.1:57782 [Receiving block BP-935274450-172.31.2.20-1700357170119:blk_1073741855_1031] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7ab9f512-shared-pool-15 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-54279523_16 at /127.0.0.1:56414 [Waiting for operation #5] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-935274450-172.31.2.20-1700357170119:blk_1073741856_1032, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_952018606_16 at /127.0.0.1:57826 [Waiting for operation #34] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7ab9f512-shared-pool-12 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/cluster_6f56ccd7-1fd7-2f9a-c808-0f86c08a2321/dfs/data/data2/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/cluster_6f56ccd7-1fd7-2f9a-c808-0f86c08a2321/dfs/data/data1/current sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7ab9f512-shared-pool-11 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7ab9f512-shared-pool-13 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7ab9f512-shared-pool-16 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: (jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: Close-WAL-Writer-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-935274450-172.31.2.20-1700357170119:blk_1073741855_1031, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7ab9f512-shared-pool-14 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1d999fb5-shared-pool-6 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-1 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=698 (was 689) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=158 (was 211), ProcessCount=168 (was 171), AvailableMemoryMB=4573 (was 4098) - AvailableMemoryMB LEAK? - 2023-11-19T01:26:55,252 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=600 is superior to 500 2023-11-19T01:26:55,271 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleOverwrite Thread=600, OpenFileDescriptor=698, MaxFileDescriptor=60000, SystemLoadAverage=158, ProcessCount=168, AvailableMemoryMB=4572 2023-11-19T01:26:55,271 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=600 is superior to 500 2023-11-19T01:26:55,271 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestFullRestore(173): test full restore on a single table empty table 2023-11-19T01:26:55,273 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5045b71f to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:26:55,285 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fbfa9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:26:55,289 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:26:55,291 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:50528, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:26:55,293 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:26:55,298 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43218, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:26:55,319 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:26:55,328 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:26:55,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357215328 (current time:1700357215328). 2023-11-19T01:26:55,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:26:55,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:26:55,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x6d691ad6 to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:26:55,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dac3048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:26:55,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:26:55,349 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:50536, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:26:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x6d691ad6 to 127.0.0.1:57501 2023-11-19T01:26:55,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:26:55,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:26:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=50, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:26:55,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-11-19T01:26:55,355 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:26:55,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-19T01:26:55,355 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:26:55,358 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:26:55,361 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:26:55,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-19T01:26:55,478 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:test-17003571846791' 2023-11-19T01:26:55,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-19T01:26:55,770 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:26:55,770 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc}] 2023-11-19T01:26:55,772 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:26:55,923 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:55,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=51 2023-11-19T01:26:55,924 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:26:55,924 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2819): Flushing 322466352cac3ade468ab2ce8e706acc 2/2 column families, dataSize=1.51 KB heapSize=2.81 KB 2023-11-19T01:26:55,940 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/5c700b4d765d4c26bf3c25f07d229757 is 171, key is trslm:hdfs://localhost:42739/backupUT\x00ns2:test-17003571846791/meta:log-roll-map/1700357207381/Put/seqid=0 2023-11-19T01:26:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-19T01:26:56,347 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=792 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/5c700b4d765d4c26bf3c25f07d229757 2023-11-19T01:26:56,357 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/fde2616a711d412499f12db80d1c34ff is 458, key is session:backup_1700357192923/session:context/1700357208254/Put/seqid=0 2023-11-19T01:26:56,363 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=754 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/fde2616a711d412499f12db80d1c34ff 2023-11-19T01:26:56,371 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/5c700b4d765d4c26bf3c25f07d229757 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757 2023-11-19T01:26:56,378 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757, entries=6, sequenceid=17, filesize=5.8 K 2023-11-19T01:26:56,379 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/fde2616a711d412499f12db80d1c34ff as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff 2023-11-19T01:26:56,386 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff, entries=2, sequenceid=17, filesize=5.4 K 2023-11-19T01:26:56,387 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(3022): Finished flush of dataSize ~1.51 KB/1546, heapSize ~2.78 KB/2848, currentSize=0 B/0 for 322466352cac3ade468ab2ce8e706acc in 463ms, sequenceid=17, compaction requested=false 2023-11-19T01:26:56,387 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.HRegion(2520): Flush status journal for 322466352cac3ade468ab2ce8e706acc: 2023-11-19T01:26:56,387 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. for snapshot_backup_system completed. 2023-11-19T01:26:56,388 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:26:56,388 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:26:56,388 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757] hfiles 2023-11-19T01:26:56,388 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757 for snapshot=snapshot_backup_system 2023-11-19T01:26:56,388 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff] hfiles 2023-11-19T01:26:56,388 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99 for snapshot=snapshot_backup_system 2023-11-19T01:26:56,388 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff for snapshot=snapshot_backup_system 2023-11-19T01:26:56,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-19T01:26:56,799 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:26:56,799 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=51 2023-11-19T01:26:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=51 2023-11-19T01:26:56,800 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:26:56,801 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=51, ppid=50, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:26:56,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=51, resume processing ppid=50 2023-11-19T01:26:56,811 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:26:56,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=51, ppid=50, state=SUCCESS; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc in 1.0370 sec 2023-11-19T01:26:56,812 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:26:56,813 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:26:56,813 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:26:56,814 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:26:57,246 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:26:57,254 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:26:57,255 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:26:57,256 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=50, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:26:57,256 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 50 2023-11-19T01:26:57,257 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=50, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=50, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.9050 sec 2023-11-19T01:26:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=50 2023-11-19T01:26:57,463 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 50 completed 2023-11-19T01:26:57,463 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(115): Backup backup_1700357215286 started at 1700357217463. 2023-11-19T01:26:57,465 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(119): Backup session backup_1700357215286 has been started. 2023-11-19T01:26:57,470 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-19T01:26:57,471 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:26:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:26:57,471 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:26:57,471 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:26:57,472 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:26:57,472 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:26:57,473 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-19T01:26:57,473 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:26:57,474 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:57,474 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-19T01:26:57,474 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:26:57,474 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:26:57,475 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:57,475 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:26:57,475 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:26:57,475 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-19T01:26:57,476 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:26:57,476 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:26:57,476 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:26:57,476 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:26:57,476 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:26:57,476 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:26:57,476 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:26:57,477 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:26:57,477 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:26:57,477 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:26:57,479 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:57,479 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:26:57,479 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:57,479 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:26:57,479 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:26:57,480 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-11-19T01:26:57,480 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:26:57,480 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:26:57,480 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:26:57,480 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:26:57,481 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:57,481 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:26:57,482 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:26:57,482 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@29393825[Count = 0] remaining members to acquire global barrier 2023-11-19T01:26:57,482 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:26:57,482 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:26:57,483 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-19T01:26:57,483 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:26:57,483 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:26:57,483 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:26:57,483 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:57,483 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:26:57,483 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:26:57,483 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:57,484 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357198565 highest: 1700357198565 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:57,484 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784:(num 1700357198565) roll requested 2023-11-19T01:26:57,496 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357217484, exclude list is [], retry=0 2023-11-19T01:26:57,500 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:26:57,501 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357198565 with entries=19, filesize=6.26 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357217484 2023-11-19T01:26:57,501 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:26:57,501 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357198565 is not closed yet, will try archiving it next time 2023-11-19T01:26:57,502 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta:.meta(num 1700357198595) roll requested 2023-11-19T01:26:57,504 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357198565 not finished, retry = 0 2023-11-19T01:26:57,513 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357217502.meta, exclude list is [], retry=0 2023-11-19T01:26:57,516 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:26:57,518 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357198595.meta with entries=24, filesize=6.26 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357217502.meta 2023-11-19T01:26:57,518 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:26:57,518 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357198595.meta is not closed yet, will try archiving it next time 2023-11-19T01:26:57,524 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-19T01:26:57,524 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357217484 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:57,534 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:26:57,534 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,44997,1700357176784' in zk 2023-11-19T01:26:57,541 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:57,541 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:26:57,541 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:26:57,541 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:26:57,542 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:57,542 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:26:57,542 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:26:57,543 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:26:57,543 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:26:57,543 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:26:57,544 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:57,544 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:26:57,544 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:26:57,545 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:57,546 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,44997,1700357176784': 2023-11-19T01:26:57,546 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:26:57,546 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:26:57,546 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:26:57,546 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:26:57,546 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:26:57,546 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:26:57,548 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-19T01:26:57,548 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-19T01:26:57,548 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:26:57,548 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-19T01:26:57,548 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:26:57,548 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:26:57,548 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:26:57,548 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:26:57,549 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:26:57,549 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:26:57,549 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:26:57,549 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:57,549 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:26:57,549 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:26:57,550 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:26:57,550 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:26:57,551 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:57,551 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:57,551 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:26:57,552 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:26:57,552 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:57,555 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:57,555 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-19T01:26:57,555 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-19T01:26:57,555 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-19T01:26:57,555 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:57,555 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-19T01:26:57,555 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:26:57,556 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-19T01:26:57,556 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:26:57,556 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-19T01:26:57,556 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-19T01:26:57,556 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:26:57,556 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:26:57,556 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:26:57,557 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:26:57,557 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:26:57,557 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin(2906): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-11-19T01:26:57,557 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin(2914): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-11-19T01:26:57,657 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin(2920): Getting current status of procedure from master... 2023-11-19T01:26:57,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-19T01:26:57,661 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357217660_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } 2023-11-19T01:26:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357217661 (current time:1700357217661). 2023-11-19T01:26:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:26:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357217660_default_test-1700357184679 VERSION not specified, setting to 2 2023-11-19T01:26:57,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x07ceb27f to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:26:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b4cecbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:26:57,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:26:57,671 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:50544, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:26:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x07ceb27f to 127.0.0.1:57501 2023-11-19T01:26:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:26:57,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:26:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=52, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1700357217660_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } 2023-11-19T01:26:57,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357217660_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-11-19T01:26:57,676 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1700357217660_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:26:57,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-19T01:26:57,677 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1700357217660_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:26:57,679 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357217660_default_test-1700357184679 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:26:57,681 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1700357217660_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:26:57,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-19T01:26:57,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-19T01:26:58,088 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1700357217660_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:26:58,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 2c9c20d76de40bb8afca0e6f5e27adb8}] 2023-11-19T01:26:58,090 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 2c9c20d76de40bb8afca0e6f5e27adb8 2023-11-19T01:26:58,241 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:26:58,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=53 2023-11-19T01:26:58,243 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8. 2023-11-19T01:26:58,244 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2819): Flushing 2c9c20d76de40bb8afca0e6f5e27adb8 1/1 column families, dataSize=3.17 KB heapSize=11 KB 2023-11-19T01:26:58,265 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8/.tmp/f/c1ff0a74052c49378feee44b0bb4ee87 is 37, key is row10/f:q1/1700357187320/Put/seqid=0 2023-11-19T01:26:58,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-19T01:26:58,683 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.17 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8/.tmp/f/c1ff0a74052c49378feee44b0bb4ee87 2023-11-19T01:26:58,691 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8/.tmp/f/c1ff0a74052c49378feee44b0bb4ee87 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8/f/c1ff0a74052c49378feee44b0bb4ee87 2023-11-19T01:26:58,698 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8/f/c1ff0a74052c49378feee44b0bb4ee87, entries=99, sequenceid=103, filesize=8.2 K 2023-11-19T01:26:58,699 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(3022): Finished flush of dataSize ~3.17 KB/3247, heapSize ~10.98 KB/11248, currentSize=0 B/0 for 2c9c20d76de40bb8afca0e6f5e27adb8 in 455ms, sequenceid=103, compaction requested=false 2023-11-19T01:26:58,700 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.HRegion(2520): Flush status journal for 2c9c20d76de40bb8afca0e6f5e27adb8: 2023-11-19T01:26:58,700 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8. for snapshot_1700357217660_default_test-1700357184679 completed. 2023-11-19T01:26:58,700 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(240): Storing 'test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8.' region-info for snapshot=snapshot_1700357217660_default_test-1700357184679 2023-11-19T01:26:58,700 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:26:58,700 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8/f/c1ff0a74052c49378feee44b0bb4ee87] hfiles 2023-11-19T01:26:58,700 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8/f/c1ff0a74052c49378feee44b0bb4ee87 for snapshot=snapshot_1700357217660_default_test-1700357184679 2023-11-19T01:26:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-19T01:26:59,106 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8. 2023-11-19T01:26:59,106 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=53 2023-11-19T01:26:59,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=53 2023-11-19T01:26:59,107 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357217660_default_test-1700357184679 on region 2c9c20d76de40bb8afca0e6f5e27adb8 2023-11-19T01:26:59,108 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=53, ppid=52, state=RUNNABLE; SnapshotRegionProcedure 2c9c20d76de40bb8afca0e6f5e27adb8 2023-11-19T01:26:59,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=53, resume processing ppid=52 2023-11-19T01:26:59,111 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1700357217660_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:26:59,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=53, ppid=52, state=SUCCESS; SnapshotRegionProcedure 2c9c20d76de40bb8afca0e6f5e27adb8 in 1.0210 sec 2023-11-19T01:26:59,112 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1700357217660_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:26:59,112 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1700357217660_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:26:59,112 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357217660_default_test-1700357184679 2023-11-19T01:26:59,113 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357217660_default_test-1700357184679 2023-11-19T01:26:59,487 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:26:59,526 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1700357217660_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:26:59,533 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1700357217660_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:26:59,533 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357217660_default_test-1700357184679 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357217660_default_test-1700357184679 2023-11-19T01:26:59,535 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=52, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1700357217660_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:26:59,535 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357217660_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 }, snapshot procedure id = 52 2023-11-19T01:26:59,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=52, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=52, snapshot={ ss=snapshot_1700357217660_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } in 1.8620 sec 2023-11-19T01:26:59,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=52 2023-11-19T01:26:59,783 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: default:test-1700357184679, procId: 52 completed 2023-11-19T01:26:59,783 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1700357215286 2023-11-19T01:26:59,783 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-19T01:26:59,783 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-19T01:26:59,783 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357215286_test-1700357184679 2023-11-19T01:26:59,783 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357217660_default_test-1700357184679 to hdfs://localhost:42739/backupUT/backup_1700357215286/default/test-1700357184679/ 2023-11-19T01:26:59,783 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:26:59,819 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:26:59,819 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,44889,1700357173572_-407220602_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42739/backupUT/backup_1700357215286/default/test-1700357184679, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42739/backupUT/backup_1700357215286/default/test-1700357184679/.hbase-snapshot/.tmp/snapshot_1700357217660_default_test-1700357184679 2023-11-19T01:26:59,819 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af, snapshotDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357217660_default_test-1700357184679. 2023-11-19T01:26:59,830 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357217660_default_test-1700357184679 to hdfs://localhost:42739/backupUT/backup_1700357215286/default/test-1700357184679/.hbase-snapshot/.tmp/snapshot_1700357217660_default_test-1700357184679 2023-11-19T01:27:00,250 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:00,251 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:00,252 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:00,252 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:00,288 INFO [master/jenkins-hbase6:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2023-11-19T01:27:00,288 INFO [master/jenkins-hbase6:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2023-11-19T01:27:01,261 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-8064694356075429799.jar 2023-11-19T01:27:01,262 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:01,262 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:01,263 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:01,263 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:01,264 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:01,265 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:01,265 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:01,266 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:01,267 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:01,268 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:27:01,268 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:27:01,269 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:27:01,269 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:27:01,270 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:27:01,270 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:27:01,271 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:27:01,271 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:27:01,272 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:27:01,272 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:27:01,272 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:27:01,273 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:27:01,273 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:27:01,274 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:27:01,275 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:27:01,275 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:27:01,275 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:27:01,276 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:27:01,277 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:27:01,313 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-19T01:27:01,334 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357217660_default_test-1700357184679' hfile list 2023-11-19T01:27:01,337 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(674): export split=0 size=8.2 K 2023-11-19T01:27:01,423 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357221379/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:27:01,423 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357221379/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:27:01,484 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-19T01:27:01,511 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8/f/c1ff0a74052c49378feee44b0bb4ee87 output=hdfs://localhost:42739/backupUT/backup_1700357215286/default/test-1700357184679/archive/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8/f/c1ff0a74052c49378feee44b0bb4ee87 2023-11-19T01:27:01,511 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8396 (8.2 K) time=0sec 4.004M/sec 2023-11-19T01:27:02,464 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:27:02,465 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:27:02,487 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357217660_default_test-1700357184679 2023-11-19T01:27:02,487 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357217660_default_test-1700357184679 finished. 2023-11-19T01:27:02,487 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1034): test-1700357184679 2023-11-19T01:27:02,502 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357215286 2023-11-19T01:27:02,503 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:27:02,917 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42739/backupUT/backup_1700357215286/.backup.manifest 2023-11-19T01:27:02,918 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(358): Backup backup_1700357215286 finished: type=FULL,tablelist=test-1700357184679,targetRootDir=hdfs://localhost:42739/backupUT,startts=1700357217463,completets=1700357222502,bytescopied=0 2023-11-19T01:27:02,918 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-19T01:27:02,918 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357217660_default_test-1700357184679 2023-11-19T01:27:02,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357217660_default_test-1700357184679" 2023-11-19T01:27:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357217660_default_test-1700357184679 2023-11-19T01:27:02,923 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357217660_default_test-1700357184679 for backup backup_1700357215286 succeeded. 2023-11-19T01:27:02,924 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:27:02,929 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:27:02,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:27:02,933 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:27:02,935 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:27:02,941 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(376): Backup backup_1700357215286 completed. 2023-11-19T01:27:02,941 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:27:02,941 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5045b71f to 127.0.0.1:57501 2023-11-19T01:27:02,942 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:02,960 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestFullRestore(178): backup complete 2023-11-19T01:27:02,964 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357215286/.backup.manifest 2023-11-19T01:27:02,966 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357215286/.backup.manifest 2023-11-19T01:27:02,967 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357215286/.backup.manifest 2023-11-19T01:27:02,969 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357215286/.backup.manifest 2023-11-19T01:27:02,970 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(148): Restoring 'test-1700357184679' to 'test-1700357184679' from full backup image hdfs://localhost:42739/backupUT/backup_1700357215286/default/test-1700357184679 2023-11-19T01:27:02,980 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(476): Truncating exising target table 'test-1700357184679', preserving region splits 2023-11-19T01:27:02,981 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$18(967): Started disable of test-1700357184679 2023-11-19T01:27:02,981 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable test-1700357184679 2023-11-19T01:27:02,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=test-1700357184679 2023-11-19T01:27:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-19T01:27:02,986 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1700357184679","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357222986"}]},"ts":"1700357222986"} 2023-11-19T01:27:02,987 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1700357184679, state=DISABLING in hbase:meta 2023-11-19T01:27:02,990 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set test-1700357184679 to state=DISABLING 2023-11-19T01:27:02,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1700357184679, region=2c9c20d76de40bb8afca0e6f5e27adb8, UNASSIGN}] 2023-11-19T01:27:02,992 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=55, ppid=54, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1700357184679, region=2c9c20d76de40bb8afca0e6f5e27adb8, UNASSIGN 2023-11-19T01:27:02,993 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=2c9c20d76de40bb8afca0e6f5e27adb8, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:02,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; CloseRegionProcedure 2c9c20d76de40bb8afca0e6f5e27adb8, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:27:03,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-19T01:27:03,146 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:03,147 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(119): Close 2c9c20d76de40bb8afca0e6f5e27adb8 2023-11-19T01:27:03,147 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1668): Closing 2c9c20d76de40bb8afca0e6f5e27adb8, disabling compactions & flushes 2023-11-19T01:27:03,147 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1690): Closing region test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8. 2023-11-19T01:27:03,147 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8. 2023-11-19T01:27:03,147 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1778): Acquired close lock on test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8. after waiting 0 ms 2023-11-19T01:27:03,147 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1788): Updates disabled for region test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8. 2023-11-19T01:27:03,154 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8/recovered.edits/106.seqid, newMaxSeqId=106, maxSeqId=1 2023-11-19T01:27:03,155 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:27:03,155 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1909): Closed test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8. 2023-11-19T01:27:03,155 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] regionserver.HRegion(1622): Region close journal for 2c9c20d76de40bb8afca0e6f5e27adb8: 2023-11-19T01:27:03,157 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=56}] handler.UnassignRegionHandler(163): Closed 2c9c20d76de40bb8afca0e6f5e27adb8 2023-11-19T01:27:03,158 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=55 updating hbase:meta row=2c9c20d76de40bb8afca0e6f5e27adb8, regionState=CLOSED 2023-11-19T01:27:03,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=56, resume processing ppid=55 2023-11-19T01:27:03,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=56, ppid=55, state=SUCCESS; CloseRegionProcedure 2c9c20d76de40bb8afca0e6f5e27adb8, server=jenkins-hbase6.apache.org,44997,1700357176784 in 164 msec 2023-11-19T01:27:03,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=55, resume processing ppid=54 2023-11-19T01:27:03,162 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=55, ppid=54, state=SUCCESS; TransitRegionStateProcedure table=test-1700357184679, region=2c9c20d76de40bb8afca0e6f5e27adb8, UNASSIGN in 171 msec 2023-11-19T01:27:03,163 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1700357184679","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357223163"}]},"ts":"1700357223163"} 2023-11-19T01:27:03,164 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1700357184679, state=DISABLED in hbase:meta 2023-11-19T01:27:03,166 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set test-1700357184679 to state=DISABLED 2023-11-19T01:27:03,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=54, state=SUCCESS; DisableTableProcedure table=test-1700357184679 in 186 msec 2023-11-19T01:27:03,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=54 2023-11-19T01:27:03,289 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$TableFuture(3745): Operation: DISABLE, Table Name: default:test-1700357184679, procId: 54 completed 2023-11-19T01:27:03,289 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$16(847): Started truncating test-1700357184679 2023-11-19T01:27:03,290 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$6(2481): Client=jenkins//172.31.2.20 truncate test-1700357184679 2023-11-19T01:27:03,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=57, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=test-1700357184679 preserveSplits=true) 2023-11-19T01:27:03,293 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(87): waiting for 'test-1700357184679' regions in transition 2023-11-19T01:27:03,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=57 2023-11-19T01:27:03,296 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8 2023-11-19T01:27:03,298 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8/f, FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8/recovered.edits] 2023-11-19T01:27:03,303 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8/f/c1ff0a74052c49378feee44b0bb4ee87 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/archive/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8/f/c1ff0a74052c49378feee44b0bb4ee87 2023-11-19T01:27:03,307 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8/recovered.edits/106.seqid to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/archive/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8/recovered.edits/106.seqid 2023-11-19T01:27:03,308 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8 2023-11-19T01:27:03,308 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived test-1700357184679 regions 2023-11-19T01:27:03,313 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of test-1700357184679 from hbase:meta 2023-11-19T01:27:03,315 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'test-1700357184679' descriptor. 2023-11-19T01:27:03,316 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'test-1700357184679' from region states. 2023-11-19T01:27:03,316 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357223316"}]},"ts":"9223372036854775807"} 2023-11-19T01:27:03,318 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:27:03,318 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 2c9c20d76de40bb8afca0e6f5e27adb8, NAME => 'test-1700357184679,,1700357185162.2c9c20d76de40bb8afca0e6f5e27adb8.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:27:03,318 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'test-1700357184679' as deleted. 2023-11-19T01:27:03,318 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1700357184679","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1700357223318"}]},"ts":"9223372036854775807"} 2023-11-19T01:27:03,320 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table test-1700357184679 state from META 2023-11-19T01:27:03,364 INFO [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 98c2f2b8120be772f779a6f83df59517, NAME => 'test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1700357184679', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:27:03,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=57 2023-11-19T01:27:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=57 2023-11-19T01:27:03,772 DEBUG [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(888): Instantiated test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:27:03,772 DEBUG [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1668): Closing 98c2f2b8120be772f779a6f83df59517, disabling compactions & flushes 2023-11-19T01:27:03,772 INFO [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1690): Closing region test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. 2023-11-19T01:27:03,772 DEBUG [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. 2023-11-19T01:27:03,772 DEBUG [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. after waiting 0 ms 2023-11-19T01:27:03,772 DEBUG [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. 2023-11-19T01:27:03,772 INFO [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1909): Closed test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. 2023-11-19T01:27:03,772 DEBUG [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1622): Region close journal for 98c2f2b8120be772f779a6f83df59517: 2023-11-19T01:27:03,774 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1700357223773"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357223773"}]},"ts":"1700357223773"} 2023-11-19T01:27:03,775 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:27:03,776 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1700357184679","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357223776"}]},"ts":"1700357223776"} 2023-11-19T01:27:03,777 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1700357184679, state=ENABLING in hbase:meta 2023-11-19T01:27:03,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1700357184679, region=98c2f2b8120be772f779a6f83df59517, ASSIGN}] 2023-11-19T01:27:03,784 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1700357184679, region=98c2f2b8120be772f779a6f83df59517, ASSIGN 2023-11-19T01:27:03,785 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1700357184679, region=98c2f2b8120be772f779a6f83df59517, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,44997,1700357176784; forceNewPlan=false, retain=false 2023-11-19T01:27:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=57 2023-11-19T01:27:03,937 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=98c2f2b8120be772f779a6f83df59517, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:03,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE; OpenRegionProcedure 98c2f2b8120be772f779a6f83df59517, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:27:04,090 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:04,094 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(130): Open test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. 2023-11-19T01:27:04,094 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7254): Opening region: {ENCODED => 98c2f2b8120be772f779a6f83df59517, NAME => 'test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:27:04,095 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:27:04,095 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1700357184679 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:04,095 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(888): Instantiated test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:27:04,095 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7296): checking encryption for 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:04,095 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7299): checking classloading for 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:04,096 INFO [StoreOpener-98c2f2b8120be772f779a6f83df59517-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:04,098 INFO [StoreOpener-98c2f2b8120be772f779a6f83df59517-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98c2f2b8120be772f779a6f83df59517 columnFamilyName f 2023-11-19T01:27:04,098 DEBUG [StoreOpener-98c2f2b8120be772f779a6f83df59517-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:27:04,098 INFO [StoreOpener-98c2f2b8120be772f779a6f83df59517-1 {}] regionserver.HStore(324): Store=98c2f2b8120be772f779a6f83df59517/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:27:04,099 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:04,100 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:04,102 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1076): writing seq id for 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:04,104 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:27:04,105 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): Opened 98c2f2b8120be772f779a6f83df59517; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9981953280, jitterRate=-0.07035815715789795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:27:04,105 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(992): Region open journal for 98c2f2b8120be772f779a6f83df59517: 2023-11-19T01:27:04,106 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517., pid=59, masterSystemTime=1700357224090 2023-11-19T01:27:04,108 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. 2023-11-19T01:27:04,108 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(158): Opened test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. 2023-11-19T01:27:04,108 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=58 updating hbase:meta row=98c2f2b8120be772f779a6f83df59517, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:04,112 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=59, resume processing ppid=58 2023-11-19T01:27:04,112 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=59, ppid=58, state=SUCCESS; OpenRegionProcedure 98c2f2b8120be772f779a6f83df59517, server=jenkins-hbase6.apache.org,44997,1700357176784 in 172 msec 2023-11-19T01:27:04,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=58, resume processing ppid=57 2023-11-19T01:27:04,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=58, ppid=57, state=SUCCESS; TransitRegionStateProcedure table=test-1700357184679, region=98c2f2b8120be772f779a6f83df59517, ASSIGN in 330 msec 2023-11-19T01:27:04,113 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1700357184679","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357224113"}]},"ts":"1700357224113"} 2023-11-19T01:27:04,114 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1700357184679, state=ENABLED in hbase:meta 2023-11-19T01:27:04,117 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(145): truncate 'test-1700357184679' completed 2023-11-19T01:27:04,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=57, state=SUCCESS; TruncateTableProcedure (table=test-1700357184679 preserveSplits=true) in 827 msec 2023-11-19T01:27:04,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=57 2023-11-19T01:27:04,399 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$TableFuture(3745): Operation: TRUNCATE, Table Name: default:test-1700357184679, procId: 57 completed 2023-11-19T01:27:04,402 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:42739/backupUT/backup_1700357215286/default/test-1700357184679/archive/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8 from hbase tables test-1700357184679 to tables test-1700357184679 2023-11-19T01:27:04,402 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1700357184679 into test-1700357184679 2023-11-19T01:27:04,404 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-test-1700357184679-1700357224402 from hdfs://localhost:42739/backupUT/backup_1700357215286/default/test-1700357184679/archive/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8 2023-11-19T01:27:04,405 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x3f0f2b08 to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:27:04,412 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4886f791, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:04,413 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:27:04,414 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:37356, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:27:04,415 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-19T01:27:04,415 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1700357184679 2023-11-19T01:27:04,418 DEBUG [hconnection-0x29f1b294-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:04,420 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:39266, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:04,421 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1700357184679: 2023-11-19T01:27:04,421 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-11-19T01:27:04,421 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/partitions_ba45362e-878b-4742-b0b8-1896a2b7bc8a 2023-11-19T01:27:04,784 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns3_table_test-17003571846792 2023-11-19T01:27:04,784 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns3_table_test-17003571846792 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:27:04,828 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:04,829 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:04,829 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:04,830 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:05,864 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-8316714310250155158.jar 2023-11-19T01:27:05,865 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:05,865 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:05,866 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:05,866 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:05,867 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:05,867 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:05,867 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:05,868 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:05,868 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:05,869 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:27:05,870 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:27:05,870 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:27:05,871 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:27:05,871 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:27:05,872 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:27:05,872 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:27:05,873 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:27:05,873 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:27:05,874 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:27:05,874 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:27:05,874 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:27:05,875 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:05,875 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:05,876 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:05,876 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:05,876 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:05,877 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:05,877 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:27:05,878 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: test-1700357184679 2023-11-19T01:27:05,878 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:27:05,878 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x3f0f2b08 to 127.0.0.1:57501 2023-11-19T01:27:05,878 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:05,878 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-11-19T01:27:05,878 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-11-19T01:27:05,879 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:27:05,886 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-19T01:27:05,892 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-11-19T01:27:05,969 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357225935/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:27:05,970 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357225935/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:27:06,032 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:42739/backupUT/backup_1700357215286/default/test-1700357184679/archive/data/default/test-1700357184679/2c9c20d76de40bb8afca0e6f5e27adb8/f/c1ff0a74052c49378feee44b0bb4ee87 2023-11-19T01:27:06,036 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-11-19T01:27:06,060 DEBUG [pool-237-thread-1 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x449378fe to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:27:06,065 DEBUG [pool-237-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@648ef90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:06,067 DEBUG [hconnection-0x158331d4-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:06,069 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:39272, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:06,073 DEBUG [pool-237-thread-1 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x449378fe to 127.0.0.1:57501 2023-11-19T01:27:06,073 DEBUG [pool-237-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:06,073 DEBUG [pool-237-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-11-19T01:27:06,073 DEBUG [pool-237-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase6.apache.org 2023-11-19T01:27:06,086 DEBUG [pool-237-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-test-1700357184679-1700357224402/_temporary/0/_temporary/attempt_local503803971_0005_r_000000_0/f/1c0274030b954243a5399227e3eb68c0 is 37, key is row10/f:q1/1700357187320/Put/seqid=0 2023-11-19T01:27:06,089 WARN [Thread-952 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase6.apache.org:44997] Specified favored nodes: [jenkins-hbase6.apache.org:44997] 2023-11-19T01:27:07,039 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-test-1700357184679-1700357224402 2023-11-19T01:27:07,040 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x5e8a406f to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:27:07,053 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32abf366, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:07,056 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:07,057 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:39278, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:07,065 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-test-1700357184679-1700357224402/_SUCCESS 2023-11-19T01:27:07,069 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:27:07,070 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:37358, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:27:07,080 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-test-1700357184679-1700357224402/f/1c0274030b954243a5399227e3eb68c0 first=Optional[row0] last=Optional[row98] 2023-11-19T01:27:07,082 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517., hostname=jenkins-hbase6.apache.org,44997,1700357176784, seqNum=2 for row with hfile group [{f,hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-test-1700357184679-1700357224402/f/1c0274030b954243a5399227e3eb68c0}] 2023-11-19T01:27:07,087 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-test-1700357184679-1700357224402/f/1c0274030b954243a5399227e3eb68c0 for inclusion in 98c2f2b8120be772f779a6f83df59517/f 2023-11-19T01:27:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-11-19T01:27:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(625): Region bounds: first= last= 2023-11-19T01:27:07,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegion(2520): Flush status journal for 98c2f2b8120be772f779a6f83df59517: 2023-11-19T01:27:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-test-1700357184679-1700357224402/f/1c0274030b954243a5399227e3eb68c0 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__test-1700357184679__li50uhvepbnerti93qg340973t5dvi7hrm9obiqjvqjpqv6e3f493r01tqjg4rha/f/1c0274030b954243a5399227e3eb68c0 2023-11-19T01:27:07,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__test-1700357184679__li50uhvepbnerti93qg340973t5dvi7hrm9obiqjvqjpqv6e3f493r01tqjg4rha/f/1c0274030b954243a5399227e3eb68c0 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ 2023-11-19T01:27:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x50897aac to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:27:07,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e8f6e6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:27:07,104 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:37368, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:27:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:07,109 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:39294, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(428): write bulk load descriptor to backup test-1700357184679 with 1 entries 2023-11-19T01:27:07,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(1602): writing raw bulk path hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ for test-1700357184679 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(435): written 1 rows for bulk load of test-1700357184679 2023-11-19T01:27:07,133 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:27:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x50897aac to 127.0.0.1:57501 2023-11-19T01:27:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:07,134 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__test-1700357184679__li50uhvepbnerti93qg340973t5dvi7hrm9obiqjvqjpqv6e3f493r01tqjg4rha/f/1c0274030b954243a5399227e3eb68c0 into 98c2f2b8120be772f779a6f83df59517/f as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ - updating store file list. 2023-11-19T01:27:07,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ into 98c2f2b8120be772f779a6f83df59517/f 2023-11-19T01:27:07,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__test-1700357184679__li50uhvepbnerti93qg340973t5dvi7hrm9obiqjvqjpqv6e3f493r01tqjg4rha/f/1c0274030b954243a5399227e3eb68c0 into 98c2f2b8120be772f779a6f83df59517/f (new location: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_) 2023-11-19T01:27:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__test-1700357184679__li50uhvepbnerti93qg340973t5dvi7hrm9obiqjvqjpqv6e3f493r01tqjg4rha/f/1c0274030b954243a5399227e3eb68c0 2023-11-19T01:27:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x461234a8 to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:27:07,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5652b5e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:27:07,157 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:37376, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-19T01:27:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:07,163 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:39302, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-11-19T01:27:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup test-1700357184679 with 1 entries 2023-11-19T01:27:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(1523): writing done bulk path hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ for test-1700357184679 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(414): written 1 rows for bulk load of test-1700357184679 2023-11-19T01:27:07,180 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:27:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x461234a8 to 127.0.0.1:57501 2023-11-19T01:27:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:07,182 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:27:07,183 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x5e8a406f to 127.0.0.1:57501 2023-11-19T01:27:07,183 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:07,183 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-11-19T01:27:07,183 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-19T01:27:07,183 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(233): Backup: backup_1700357215286 hdfs://localhost:42739/backupUT/backup_1700357215286/default/test-1700357184679/ 2023-11-19T01:27:07,183 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-11-19T01:27:07,202 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleOverwrite Thread=605 (was 600) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57501@0x461234a8 java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1517) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1541) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$537/1424953815.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x1d999fb5-shared-pool-9 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57501@0x50897aac-SendThread(127.0.0.1:57501) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1394) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1317) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57501@0x461234a8-SendThread(127.0.0.1:57501) java.lang.Thread.sleep(Native Method) org.apache.zookeeper.ClientCnxnSocketNIO.cleanup(ClientCnxnSocketNIO.java:214) org.apache.zookeeper.ClientCnxn$SendThread.cleanup(ClientCnxn.java:1394) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1317) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57501@0x5e8a406f-SendThread(127.0.0.1:57501) sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-54279523_16 at /127.0.0.1:56504 [Receiving block BP-935274450-172.31.2.20-1700357170119:blk_1073741882_1058] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57501@0x50897aac-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549) Potentially hanging thread: PacketResponder: BP-935274450-172.31.2.20-1700357170119:blk_1073741882_1058, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57501@0x5e8a406f java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1598) org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1570) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1537) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$537/1424953815.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: PacketResponder: BP-935274450-172.31.2.20-1700357170119:blk_1073741883_1059, type=LAST_IN_PIPELINE java.lang.Object.wait(Native Method) java.lang.Object.wait(Object.java:502) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1327) org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1399) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57501@0x50897aac java.lang.Object.wait(Native Method) java.lang.Thread.join(Thread.java:1257) java.lang.Thread.join(Thread.java:1331) org.apache.zookeeper.ClientCnxn.disconnect(ClientCnxn.java:1517) org.apache.zookeeper.ClientCnxn.close(ClientCnxn.java:1541) org.apache.zookeeper.ZooKeeper.close(ZooKeeper.java:1227) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.closeZk(ReadOnlyZKClient.java:306) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:356) org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$537/1424953815.run(Unknown Source) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_952018606_16 at /127.0.0.1:56536 [Waiting for operation #25] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read(BufferedInputStream.java:265) java.io.DataInputStream.readShort(DataInputStream.java:312) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:67) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:269) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57501@0x461234a8-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-54279523_16 at /127.0.0.1:56508 [Receiving block BP-935274450-172.31.2.20-1700357170119:blk_1073741883_1059] sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:157) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.io.BufferedInputStream.fill(BufferedInputStream.java:246) java.io.BufferedInputStream.read1(BufferedInputStream.java:286) java.io.BufferedInputStream.read(BufferedInputStream.java:345) java.io.DataInputStream.read(DataInputStream.java:149) org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:209) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:211) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:134) org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:109) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:528) org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:968) org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:877) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:166) org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:103) org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:290) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: HFileArchiver-3 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: hconnection-0x7ab9f512-shared-pool-23 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:57501@0x5e8a406f-EventThread sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:549) Potentially hanging thread: hconnection-0x7ab9f512-shared-pool-24 sun.misc.Unsafe.park(Native Method) java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:467) java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) java.lang.Thread.run(Thread.java:750) - Thread LEAK? -, OpenFileDescriptor=724 (was 698) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=208 (was 158) - SystemLoadAverage LEAK? -, ProcessCount=168 (was 168), AvailableMemoryMB=4055 (was 4572) 2023-11-19T01:27:07,202 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-11-19T01:27:07,217 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleDNECommand Thread=605, OpenFileDescriptor=724, MaxFileDescriptor=60000, SystemLoadAverage=208, ProcessCount=167, AvailableMemoryMB=4054 2023-11-19T01:27:07,217 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-11-19T01:27:07,217 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestFullRestore(280): test restore fails on a single table that does not exist: command-line 2023-11-19T01:27:07,218 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x4b19a344 to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:27:07,223 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422b6ddf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:07,228 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:07,229 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:39306, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:07,230 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:27:07,231 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:37384, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:27:07,242 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:27:07,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:27:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357227247 (current time:1700357227247). 2023-11-19T01:27:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:27:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:27:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x7aef0a1c to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:27:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@641e9297, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:07,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:07,257 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:39318, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x7aef0a1c to 127.0.0.1:57501 2023-11-19T01:27:07,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:07,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:27:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=60, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:27:07,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2023-11-19T01:27:07,261 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:27:07,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-19T01:27:07,261 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:27:07,264 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:27:07,266 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:27:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-19T01:27:07,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-19T01:27:07,672 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:27:07,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc}] 2023-11-19T01:27:07,676 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:27:07,827 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:07,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=61 2023-11-19T01:27:07,828 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:27:07,828 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2819): Flushing 322466352cac3ade468ab2ce8e706acc 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-11-19T01:27:07,833 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/33514f8ae1f64952ad0b4b83073e2d63 is 169, key is trslm:hdfs://localhost:42739/backupUT\x00test-1700357184679/meta:log-roll-map/1700357222491/Put/seqid=0 2023-11-19T01:27:07,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-19T01:27:08,239 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/33514f8ae1f64952ad0b4b83073e2d63 2023-11-19T01:27:08,249 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/b83bc5cc85424cf8b1d3e429f8b40460 is 452, key is session:backup_1700357215286/session:context/1700357222934/Put/seqid=0 2023-11-19T01:27:08,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-19T01:27:08,653 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=28 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/b83bc5cc85424cf8b1d3e429f8b40460 2023-11-19T01:27:08,657 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:27:08,661 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/33514f8ae1f64952ad0b4b83073e2d63 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63 2023-11-19T01:27:08,708 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63, entries=4, sequenceid=28, filesize=5.5 K 2023-11-19T01:27:08,709 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/b83bc5cc85424cf8b1d3e429f8b40460 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460 2023-11-19T01:27:08,725 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460, entries=2, sequenceid=28, filesize=5.4 K 2023-11-19T01:27:08,727 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 322466352cac3ade468ab2ce8e706acc in 898ms, sequenceid=28, compaction requested=true 2023-11-19T01:27:08,727 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.HRegion(2520): Flush status journal for 322466352cac3ade468ab2ce8e706acc: 2023-11-19T01:27:08,727 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. for snapshot_backup_system completed. 2023-11-19T01:27:08,727 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:27:08,727 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:27:08,727 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63] hfiles 2023-11-19T01:27:08,727 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (1/2): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757 for snapshot=snapshot_backup_system 2023-11-19T01:27:08,727 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (2/2): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63 for snapshot=snapshot_backup_system 2023-11-19T01:27:08,727 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460] hfiles 2023-11-19T01:27:08,727 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99 for snapshot=snapshot_backup_system 2023-11-19T01:27:08,727 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff for snapshot=snapshot_backup_system 2023-11-19T01:27:08,727 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460 for snapshot=snapshot_backup_system 2023-11-19T01:27:09,138 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:27:09,138 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=61 2023-11-19T01:27:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=61 2023-11-19T01:27:09,139 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:27:09,142 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=61, ppid=60, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:27:09,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=61, resume processing ppid=60 2023-11-19T01:27:09,144 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:27:09,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=61, ppid=60, state=SUCCESS; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc in 1.4700 sec 2023-11-19T01:27:09,145 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:27:09,145 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:27:09,145 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:27:09,146 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:27:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-19T01:27:09,557 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:27:09,567 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:27:09,568 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:27:09,569 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=60, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:27:09,569 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 60 2023-11-19T01:27:09,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=60, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=60, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.3110 sec 2023-11-19T01:27:11,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=60 2023-11-19T01:27:11,368 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: backup:system, procId: 60 completed 2023-11-19T01:27:11,368 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(115): Backup backup_1700357227223 started at 1700357231368. 2023-11-19T01:27:11,370 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(119): Backup session backup_1700357227223 has been started. 2023-11-19T01:27:11,374 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-19T01:27:11,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:27:11,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:27:11,375 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:27:11,375 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:27:11,376 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:27:11,376 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:27:11,377 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-19T01:27:11,377 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:27:11,378 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-19T01:27:11,378 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:27:11,378 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:11,378 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:27:11,379 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:11,379 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:27:11,379 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:27:11,379 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/abort/rolllog 2023-11-19T01:27:11,379 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:27:11,379 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:27:11,380 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:27:11,380 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:27:11,380 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:27:11,380 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:27:11,380 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:27:11,381 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:27:11,381 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:27:11,381 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:27:11,383 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:11,383 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:11,383 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:11,383 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:27:11,383 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:27:11,383 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog 2023-11-19T01:27:11,383 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:27:11,383 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:27:11,384 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:27:11,384 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:11,384 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:11,385 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:27:11,385 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:27:11,385 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@2ace52c5[Count = 0] remaining members to acquire global barrier 2023-11-19T01:27:11,385 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:27:11,385 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:11,387 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:11,387 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:11,387 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:11,387 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:27:11,387 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(113): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on znode that does not yet exist, /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:11,387 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:27:11,387 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:27:11,388 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:11,388 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357217484 highest: 1700357217484 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:11,388 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784:(num 1700357217484) roll requested 2023-11-19T01:27:11,400 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357231388, exclude list is [], retry=0 2023-11-19T01:27:11,403 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:27:11,405 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357217484 with entries=17, filesize=5.25 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357231388 2023-11-19T01:27:11,405 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:27:11,406 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357217484 is not closed yet, will try archiving it next time 2023-11-19T01:27:11,406 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta:.meta(num 1700357217502) roll requested 2023-11-19T01:27:11,418 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357231406.meta, exclude list is [], retry=0 2023-11-19T01:27:11,421 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:27:11,422 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357217502.meta with entries=12, filesize=3.07 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357231406.meta 2023-11-19T01:27:11,423 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:27:11,423 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357217502.meta is not closed yet, will try archiving it next time 2023-11-19T01:27:11,428 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-19T01:27:11,428 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357231388 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:11,436 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:27:11,436 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,44997,1700357176784' in zk 2023-11-19T01:27:11,438 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:27:11,438 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:11,438 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:27:11,439 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:27:11,439 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:11,439 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:27:11,439 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:27:11,440 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:27:11,440 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:27:11,440 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:11,441 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:11,441 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:27:11,441 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:11,442 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:11,442 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,44997,1700357176784': 2023-11-19T01:27:11,442 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:27:11,442 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:27:11,442 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:27:11,442 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:27:11,442 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:27:11,442 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:27:11,444 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-19T01:27:11,444 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-19T01:27:11,444 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-19T01:27:11,444 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:27:11,444 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:27:11,444 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:27:11,444 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:27:11,444 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:27:11,445 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:27:11,445 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:27:11,445 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:27:11,445 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:11,445 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:27:11,445 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:11,446 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:27:11,446 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:11,446 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:11,446 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] zookeeper.ZKUtil(111): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Set watcher on existing znode=/1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:11,447 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:27:11,447 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:11,448 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:11,451 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:11,451 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-19T01:27:11,451 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/acquired/rolllog 2023-11-19T01:27:11,451 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/acquired 2023-11-19T01:27:11,451 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:11,451 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:11,451 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:27:11,451 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:27:11,452 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): regionserver:44997-0x10123a379ab0001, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/1/rolllog-proc/abort 2023-11-19T01:27:11,452 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:27:11,452 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:27:11,453 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:11,453 DEBUG [Listener at localhost/35765-EventThread {}] zookeeper.ZKWatcher(604): master:44889-0x10123a379ab0000, quorum=127.0.0.1:57501, baseZNode=/1 Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/1/rolllog-proc/abort/rolllog 2023-11-19T01:27:11,453 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:27:11,453 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:27:11,453 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:27:11,454 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin(2906): Waiting a max of 300000 ms for procedure 'rolllog-proc : rolllog'' to complete. (max 6666 ms per retry) 2023-11-19T01:27:11,454 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin(2914): (#1) Sleeping: 100ms while waiting for procedure completion. 2023-11-19T01:27:11,554 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin(2920): Getting current status of procedure from master... 2023-11-19T01:27:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-19T01:27:11,558 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357231557_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } 2023-11-19T01:27:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357231558 (current time:1700357231558). 2023-11-19T01:27:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:27:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357231557_default_test-1700357184679 VERSION not specified, setting to 2 2023-11-19T01:27:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] zookeeper.ReadOnlyZKClient(139): Connect 0x0e514095 to 127.0.0.1:57501 with session timeout=90000ms, retries 30, retry interval 1000ms, keepAlive=60000ms 2023-11-19T01:27:11,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cec14a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:11,569 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43270, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x0e514095 to 127.0.0.1:57501 2023-11-19T01:27:11,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:11,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:27:11,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=62, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1700357231557_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } 2023-11-19T01:27:11,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357231557_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-11-19T01:27:11,573 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1700357231557_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:27:11,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-19T01:27:11,574 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1700357231557_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:27:11,577 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357231557_default_test-1700357184679 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:27:11,579 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1700357231557_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:27:11,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-19T01:27:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-19T01:27:11,985 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1700357231557_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:27:11,986 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure 98c2f2b8120be772f779a6f83df59517}] 2023-11-19T01:27:11,988 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:12,139 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:12,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=63 2023-11-19T01:27:12,139 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. 2023-11-19T01:27:12,140 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.HRegion(2520): Flush status journal for 98c2f2b8120be772f779a6f83df59517: 2023-11-19T01:27:12,140 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. for snapshot_1700357231557_default_test-1700357184679 completed. 2023-11-19T01:27:12,140 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(240): Storing 'test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517.' region-info for snapshot=snapshot_1700357231557_default_test-1700357184679 2023-11-19T01:27:12,140 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:27:12,140 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_] hfiles 2023-11-19T01:27:12,140 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ for snapshot=snapshot_1700357231557_default_test-1700357184679 2023-11-19T01:27:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-19T01:27:12,546 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. 2023-11-19T01:27:12,546 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=63 2023-11-19T01:27:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=63 2023-11-19T01:27:12,547 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357231557_default_test-1700357184679 on region 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:12,548 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=63, ppid=62, state=RUNNABLE; SnapshotRegionProcedure 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:12,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=63, resume processing ppid=62 2023-11-19T01:27:12,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=63, ppid=62, state=SUCCESS; SnapshotRegionProcedure 98c2f2b8120be772f779a6f83df59517 in 564 msec 2023-11-19T01:27:12,550 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1700357231557_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:27:12,551 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1700357231557_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:27:12,552 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1700357231557_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:27:12,552 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357231557_default_test-1700357184679 2023-11-19T01:27:12,553 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357231557_default_test-1700357184679 2023-11-19T01:27:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-19T01:27:12,969 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1700357231557_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:27:12,976 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1700357231557_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:27:12,977 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357231557_default_test-1700357184679 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357231557_default_test-1700357184679 2023-11-19T01:27:12,978 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=62, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1700357231557_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:27:12,978 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357231557_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 }, snapshot procedure id = 62 2023-11-19T01:27:12,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=62, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=62, snapshot={ ss=snapshot_1700357231557_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } in 1.4080 sec 2023-11-19T01:27:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=62 2023-11-19T01:27:13,679 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.HBaseAdmin$TableFuture(3745): Operation: SNAPSHOT, Table Name: default:test-1700357184679, procId: 62 completed 2023-11-19T01:27:13,679 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1700357227223 2023-11-19T01:27:13,679 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-19T01:27:13,680 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-19T01:27:13,680 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357227223_test-1700357184679 2023-11-19T01:27:13,680 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357231557_default_test-1700357184679 to hdfs://localhost:42739/backupUT/backup_1700357227223/default/test-1700357184679/ 2023-11-19T01:27:13,680 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:27:13,703 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:27:13,704 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,44889,1700357173572_1905769777_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42739/backupUT/backup_1700357227223/default/test-1700357184679, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42739/backupUT/backup_1700357227223/default/test-1700357184679/.hbase-snapshot/.tmp/snapshot_1700357231557_default_test-1700357184679 2023-11-19T01:27:13,704 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af, snapshotDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357231557_default_test-1700357184679. 2023-11-19T01:27:13,711 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357231557_default_test-1700357184679 to hdfs://localhost:42739/backupUT/backup_1700357227223/default/test-1700357184679/.hbase-snapshot/.tmp/snapshot_1700357231557_default_test-1700357184679 2023-11-19T01:27:14,129 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:14,130 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:14,131 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:14,131 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:15,140 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-2364781663217977990.jar 2023-11-19T01:27:15,140 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:15,141 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:15,141 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:15,141 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:15,142 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:15,143 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:15,143 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:15,144 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:15,144 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:15,145 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:27:15,146 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:27:15,146 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:27:15,146 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:27:15,147 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:27:15,147 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:27:15,148 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:27:15,148 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:27:15,149 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:27:15,149 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:27:15,150 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:27:15,150 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:27:15,151 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:27:15,151 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:27:15,152 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:27:15,152 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:27:15,153 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:27:15,153 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:27:15,154 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:27:15,181 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.JobResourceUploader(142): Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this. 2023-11-19T01:27:15,200 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357231557_default_test-1700357184679' hfile list 2023-11-19T01:27:15,202 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-19T01:27:15,274 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357235240/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:27:15,274 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357235240/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:27:15,329 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-19T01:27:15,353 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ output=hdfs://localhost:42739/backupUT/backup_1700357227223/default/test-1700357184679/archive/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ 2023-11-19T01:27:15,353 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-11-19T01:27:16,311 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:27:16,312 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:27:16,318 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357231557_default_test-1700357184679 2023-11-19T01:27:16,318 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357231557_default_test-1700357184679 finished. 2023-11-19T01:27:16,318 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1034): test-1700357184679 2023-11-19T01:27:16,324 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357227223 2023-11-19T01:27:16,325 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:27:16,730 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42739/backupUT/backup_1700357227223/.backup.manifest 2023-11-19T01:27:16,730 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(358): Backup backup_1700357227223 finished: type=FULL,tablelist=test-1700357184679,targetRootDir=hdfs://localhost:42739/backupUT,startts=1700357231368,completets=1700357236324,bytescopied=0 2023-11-19T01:27:16,730 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-19T01:27:16,730 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357231557_default_test-1700357184679 2023-11-19T01:27:16,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357231557_default_test-1700357184679" 2023-11-19T01:27:16,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357231557_default_test-1700357184679 2023-11-19T01:27:16,734 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357231557_default_test-1700357184679 for backup backup_1700357227223 succeeded. 2023-11-19T01:27:16,735 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:27:16,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:27:16,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:27:16,742 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:27:16,744 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:27:16,749 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(376): Backup backup_1700357227223 completed. 2023-11-19T01:27:16,749 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] client.ConnectionImplementation(2126): Closing master protocol: MasterService 2023-11-19T01:27:16,749 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] zookeeper.ReadOnlyZKClient(365): Close zookeeper connection 0x4b19a344 to 127.0.0.1:57501 2023-11-19T01:27:16,749 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:16,760 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestFullRestore(285): backup complete You have to specify either set name or table list to restore Usage: hbase restore [options] backup_path Path to a backup destination root backup_id Backup image ID to restore table(s) Comma-separated list of tables to restore Options: -c Check restore sequence and dependencies only (does not execute the command) -d Enable debug loggings -h,--help Show usage -m A comma separated list of target tables. If specified, each table in must have a mapping -o Overwrite data if any of the restore target tables exists -q Yarn queue name to run backup restore command on -s Backup set to restore, mutually exclusive with -t (table list) -t Table name list, comma-separated. To enable backup, in hbase-site.xml, set: hbase.backup.enable=true hbase.master.logcleaner.plugins=YOUR_PLUGINS,org.apache.hadoop.hbase.backup.master.BackupLogCleaner hbase.procedure.master.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager hbase.procedure.regionserver.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager hbase.coprocessor.region.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.BackupObserver and restart the cluster For more information please see http://hbase.apache.org/book.html#backuprestore 2023-11-19T01:27:16,790 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleDNECommand Thread=597 (was 605), OpenFileDescriptor=714 (was 724), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=176 (was 208), ProcessCount=166 (was 167), AvailableMemoryMB=3795 (was 4054) 2023-11-19T01:27:16,790 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=597 is superior to 500 2023-11-19T01:27:16,805 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreCheckCommand Thread=597, OpenFileDescriptor=714, MaxFileDescriptor=60000, SystemLoadAverage=176, ProcessCount=166, AvailableMemoryMB=3794 2023-11-19T01:27:16,805 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=597 is superior to 500 2023-11-19T01:27:16,805 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestFullRestore(97): test full restore on a single table: command-line, check only 2023-11-19T01:27:16,810 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eedffcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:16,814 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:16,816 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43276, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:16,817 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:27:16,819 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:34464, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:27:16,830 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:27:16,836 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:27:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357236836 (current time:1700357236836). 2023-11-19T01:27:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:27:16,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:27:16,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@110e1b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:16,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:16,846 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43292, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:27:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=64, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:27:16,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-11-19T01:27:16,850 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:27:16,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-19T01:27:16,851 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:27:16,853 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:27:16,855 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:27:16,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-19T01:27:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-19T01:27:17,261 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:27:17,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc}] 2023-11-19T01:27:17,263 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:27:17,414 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:17,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=65 2023-11-19T01:27:17,415 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:27:17,415 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2819): Flushing 322466352cac3ade468ab2ce8e706acc 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-11-19T01:27:17,419 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/2523325fca554399aaad3206b6dd67a2 is 169, key is trslm:hdfs://localhost:42739/backupUT\x00test-1700357184679/meta:log-roll-map/1700357236321/Put/seqid=0 2023-11-19T01:27:17,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-19T01:27:17,824 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/2523325fca554399aaad3206b6dd67a2 2023-11-19T01:27:17,833 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/b035097b152048f4aa18d7b48c5588c1 is 452, key is session:backup_1700357227223/session:context/1700357236743/Put/seqid=0 2023-11-19T01:27:17,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-19T01:27:18,237 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/b035097b152048f4aa18d7b48c5588c1 2023-11-19T01:27:18,244 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/2523325fca554399aaad3206b6dd67a2 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2 2023-11-19T01:27:18,250 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2, entries=4, sequenceid=39, filesize=5.5 K 2023-11-19T01:27:18,251 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/b035097b152048f4aa18d7b48c5588c1 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1 2023-11-19T01:27:18,255 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1, entries=2, sequenceid=39, filesize=5.4 K 2023-11-19T01:27:18,256 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 322466352cac3ade468ab2ce8e706acc in 841ms, sequenceid=39, compaction requested=true 2023-11-19T01:27:18,256 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.HRegion(2520): Flush status journal for 322466352cac3ade468ab2ce8e706acc: 2023-11-19T01:27:18,256 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. for snapshot_backup_system completed. 2023-11-19T01:27:18,256 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:27:18,257 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:27:18,257 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2] hfiles 2023-11-19T01:27:18,257 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/3): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757 for snapshot=snapshot_backup_system 2023-11-19T01:27:18,257 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (2/3): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63 for snapshot=snapshot_backup_system 2023-11-19T01:27:18,257 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (3/3): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2 for snapshot=snapshot_backup_system 2023-11-19T01:27:18,257 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1] hfiles 2023-11-19T01:27:18,257 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99 for snapshot=snapshot_backup_system 2023-11-19T01:27:18,257 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff for snapshot=snapshot_backup_system 2023-11-19T01:27:18,257 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460 for snapshot=snapshot_backup_system 2023-11-19T01:27:18,257 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1 for snapshot=snapshot_backup_system 2023-11-19T01:27:18,581 DEBUG [regionserver/jenkins-hbase6:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2023-11-19T01:27:18,662 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:27:18,662 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=65 2023-11-19T01:27:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=65 2023-11-19T01:27:18,662 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:27:18,664 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=65, ppid=64, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:27:18,666 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=65, resume processing ppid=64 2023-11-19T01:27:18,666 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=65, ppid=64, state=SUCCESS; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc in 1.4030 sec 2023-11-19T01:27:18,666 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:27:18,666 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:27:18,667 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:27:18,667 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:27:18,669 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:27:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-19T01:27:19,082 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:27:19,093 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:27:19,093 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:27:19,095 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=64, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:27:19,095 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 64 2023-11-19T01:27:19,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=64, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=64, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2480 sec 2023-11-19T01:27:20,059 DEBUG [master/jenkins-hbase6:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region f8f5111e949766749b5234926ea710eb changed from -1.0 to 0.0, refreshing cache 2023-11-19T01:27:20,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=64 2023-11-19T01:27:20,957 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(115): Backup backup_1700357236810 started at 1700357240957. 2023-11-19T01:27:20,959 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(119): Backup session backup_1700357236810 has been started. 2023-11-19T01:27:20,966 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-19T01:27:20,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:27:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-11-19T01:27:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:27:20,967 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:27:20,967 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:27:20,968 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:27:20,968 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:27:20,969 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:27:20,970 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:20,970 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:27:20,970 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:27:20,971 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:27:20,971 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:27:20,972 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:27:20,972 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:27:20,972 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:27:20,972 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:27:20,972 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-11-19T01:27:20,972 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:27:20,972 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:27:20,972 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:27:20,973 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:27:20,973 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:27:20,973 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:27:20,976 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:20,976 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:20,976 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:27:20,976 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:27:20,977 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:27:20,977 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:27:20,977 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:27:20,977 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:20,978 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:20,978 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:27:20,978 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:27:20,978 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@2c1fea48[Count = 0] remaining members to acquire global barrier 2023-11-19T01:27:20,978 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:27:20,978 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:20,979 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:20,979 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:20,979 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:27:20,980 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:27:20,980 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:27:20,980 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:20,980 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357231388 highest: 1700357231388 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:20,980 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784:(num 1700357231388) roll requested 2023-11-19T01:27:20,991 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357240980, exclude list is [], retry=0 2023-11-19T01:27:20,996 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:27:20,997 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357231388 with entries=10, filesize=2.31 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357240980 2023-11-19T01:27:20,997 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:27:20,997 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357231388 is not closed yet, will try archiving it next time 2023-11-19T01:27:20,997 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta:.meta(num 1700357231406) roll requested 2023-11-19T01:27:21,009 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357240997.meta, exclude list is [], retry=0 2023-11-19T01:27:21,012 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:27:21,013 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357231406.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357240997.meta 2023-11-19T01:27:21,013 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:27:21,013 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357231406.meta is not closed yet, will try archiving it next time 2023-11-19T01:27:21,016 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357231406.meta to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/oldWALs/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357231406.meta 2023-11-19T01:27:21,020 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-19T01:27:21,020 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357240980 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:21,029 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:27:21,029 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,44997,1700357176784' in zk 2023-11-19T01:27:21,031 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:27:21,031 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:27:21,031 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:21,032 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:27:21,032 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:27:21,032 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:27:21,032 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:27:21,033 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:27:21,033 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:21,033 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:21,034 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:27:21,034 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:21,034 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:21,035 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,44997,1700357176784': 2023-11-19T01:27:21,035 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:27:21,035 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:27:21,035 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:27:21,035 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:27:21,035 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:27:21,035 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:27:21,037 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:27:21,037 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:27:21,037 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:27:21,037 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:27:21,037 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:27:21,037 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:27:21,038 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:27:21,038 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:27:21,038 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:27:21,038 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:21,038 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:27:21,039 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:21,039 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:21,039 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:27:21,040 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:21,040 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:21,044 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:27:21,044 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:27:21,044 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:27:21,044 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:27:21,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:27:21,045 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:27:21,045 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:27:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-19T01:27:21,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357241148_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } 2023-11-19T01:27:21,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357241149 (current time:1700357241149). 2023-11-19T01:27:21,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:27:21,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357241148_default_test-1700357184679 VERSION not specified, setting to 2 2023-11-19T01:27:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d434b56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:21,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:21,158 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:55664, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:21,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:21,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:27:21,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=66, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357241148_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } 2023-11-19T01:27:21,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357241148_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-11-19T01:27:21,162 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357241148_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:27:21,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-19T01:27:21,163 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357241148_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:27:21,165 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357241148_default_test-1700357184679 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:27:21,167 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357241148_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:27:21,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-19T01:27:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-19T01:27:21,573 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357241148_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:27:21,573 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure 98c2f2b8120be772f779a6f83df59517}] 2023-11-19T01:27:21,575 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:21,726 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:21,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=67 2023-11-19T01:27:21,727 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. 2023-11-19T01:27:21,728 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.HRegion(2520): Flush status journal for 98c2f2b8120be772f779a6f83df59517: 2023-11-19T01:27:21,728 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. for snapshot_1700357241148_default_test-1700357184679 completed. 2023-11-19T01:27:21,728 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(240): Storing 'test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517.' region-info for snapshot=snapshot_1700357241148_default_test-1700357184679 2023-11-19T01:27:21,728 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:27:21,728 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_] hfiles 2023-11-19T01:27:21,728 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ for snapshot=snapshot_1700357241148_default_test-1700357184679 2023-11-19T01:27:21,734 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. 2023-11-19T01:27:21,734 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=67 2023-11-19T01:27:21,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=67 2023-11-19T01:27:21,735 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357241148_default_test-1700357184679 on region 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:21,736 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=67, ppid=66, state=RUNNABLE; SnapshotRegionProcedure 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:21,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=67, resume processing ppid=66 2023-11-19T01:27:21,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=67, ppid=66, state=SUCCESS; SnapshotRegionProcedure 98c2f2b8120be772f779a6f83df59517 in 164 msec 2023-11-19T01:27:21,739 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357241148_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:27:21,740 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357241148_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:27:21,740 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357241148_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:27:21,740 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357241148_default_test-1700357184679 2023-11-19T01:27:21,742 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357241148_default_test-1700357184679 2023-11-19T01:27:21,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-19T01:27:22,151 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357241148_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:27:22,158 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357241148_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:27:22,158 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357241148_default_test-1700357184679 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357241148_default_test-1700357184679 2023-11-19T01:27:22,160 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=66, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357241148_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:27:22,160 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357241148_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 }, snapshot procedure id = 66 2023-11-19T01:27:22,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=66, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=66, snapshot={ ss=snapshot_1700357241148_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } in 1.0010 sec 2023-11-19T01:27:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=66 2023-11-19T01:27:22,266 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1700357236810 2023-11-19T01:27:22,266 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-19T01:27:22,266 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-19T01:27:22,266 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357236810_test-1700357184679 2023-11-19T01:27:22,266 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357241148_default_test-1700357184679 to hdfs://localhost:42739/backupUT/backup_1700357236810/default/test-1700357184679/ 2023-11-19T01:27:22,266 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:27:22,290 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:27:22,290 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,44889,1700357173572_-1145448934_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42739/backupUT/backup_1700357236810/default/test-1700357184679, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42739/backupUT/backup_1700357236810/default/test-1700357184679/.hbase-snapshot/.tmp/snapshot_1700357241148_default_test-1700357184679 2023-11-19T01:27:22,290 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af, snapshotDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357241148_default_test-1700357184679. 2023-11-19T01:27:22,297 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357241148_default_test-1700357184679 to hdfs://localhost:42739/backupUT/backup_1700357236810/default/test-1700357184679/.hbase-snapshot/.tmp/snapshot_1700357241148_default_test-1700357184679 2023-11-19T01:27:22,713 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:22,714 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:22,714 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:22,715 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:23,733 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-3903586665477721501.jar 2023-11-19T01:27:23,734 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:23,734 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:23,735 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:23,735 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:23,735 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:23,736 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:23,736 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:23,736 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:23,737 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:23,737 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:27:23,738 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:27:23,739 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:27:23,739 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:27:23,739 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:27:23,740 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:27:23,740 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:27:23,740 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:27:23,741 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:27:23,741 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:27:23,742 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:27:23,742 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:27:23,743 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:27:23,743 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:27:23,744 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:27:23,744 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:27:23,744 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:27:23,745 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:27:23,746 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:27:23,790 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357241148_default_test-1700357184679' hfile list 2023-11-19T01:27:23,792 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-19T01:27:23,874 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357243833/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:27:23,874 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357243833/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:27:23,928 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-19T01:27:23,952 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ output=hdfs://localhost:42739/backupUT/backup_1700357236810/default/test-1700357184679/archive/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ 2023-11-19T01:27:23,952 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-11-19T01:27:24,909 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:27:24,910 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:27:24,917 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357241148_default_test-1700357184679 2023-11-19T01:27:24,917 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357241148_default_test-1700357184679 finished. 2023-11-19T01:27:24,917 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1034): test-1700357184679 2023-11-19T01:27:24,924 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357236810 2023-11-19T01:27:24,924 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:27:25,330 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42739/backupUT/backup_1700357236810/.backup.manifest 2023-11-19T01:27:25,330 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(358): Backup backup_1700357236810 finished: type=FULL,tablelist=test-1700357184679,targetRootDir=hdfs://localhost:42739/backupUT,startts=1700357240957,completets=1700357244924,bytescopied=0 2023-11-19T01:27:25,330 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-19T01:27:25,330 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357241148_default_test-1700357184679 2023-11-19T01:27:25,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357241148_default_test-1700357184679" 2023-11-19T01:27:25,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357241148_default_test-1700357184679 2023-11-19T01:27:25,334 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357241148_default_test-1700357184679 for backup backup_1700357236810 succeeded. 2023-11-19T01:27:25,335 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:27:25,339 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:27:25,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:27:25,342 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:27:25,343 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:27:25,350 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(376): Backup backup_1700357236810 completed. 2023-11-19T01:27:25,350 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:25,350 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestFullRestore(101): backup complete 2023-11-19T01:27:25,363 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] backup.RestoreDriver(108): Found -check option in restore command, will check and verify the dependencies 2023-11-19T01:27:25,374 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22cf3bf2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:25,376 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357236810/.backup.manifest 2023-11-19T01:27:25,378 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357236810/.backup.manifest 2023-11-19T01:27:25,378 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] util.BackupUtils(679): Dependent image(s) from old to new: 2023-11-19T01:27:25,379 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] util.BackupUtils(688): Backup image: backup_1700357236810 for 'test-1700357184679' is available 2023-11-19T01:27:25,379 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupAdminImpl(511): Checking backup images: OK 2023-11-19T01:27:25,379 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:25,398 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreCheckCommand Thread=596 (was 597), OpenFileDescriptor=709 (was 714), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=162 (was 176), ProcessCount=166 (was 166), AvailableMemoryMB=3771 (was 3794) 2023-11-19T01:27:25,398 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=596 is superior to 500 2023-11-19T01:27:25,413 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingle Thread=596, OpenFileDescriptor=709, MaxFileDescriptor=60000, SystemLoadAverage=162, ProcessCount=166, AvailableMemoryMB=3770 2023-11-19T01:27:25,413 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=596 is superior to 500 2023-11-19T01:27:25,413 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestFullRestore(55): test full restore on a single table empty table 2023-11-19T01:27:25,418 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b71983, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:25,420 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:25,422 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:55680, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:25,424 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:27:25,425 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:52670, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:27:25,440 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:27:25,446 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:27:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357245446 (current time:1700357245446). 2023-11-19T01:27:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:27:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:27:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14804b93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:25,455 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:55684, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:27:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=68, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:27:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 68 2023-11-19T01:27:25,459 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:27:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=68 2023-11-19T01:27:25,459 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:27:25,461 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:27:25,463 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:27:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=68 2023-11-19T01:27:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=68 2023-11-19T01:27:25,870 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:27:25,870 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc}] 2023-11-19T01:27:25,872 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=69, ppid=68, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:27:26,022 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:26,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=69 2023-11-19T01:27:26,023 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:27:26,023 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegion(2819): Flushing 322466352cac3ade468ab2ce8e706acc 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-11-19T01:27:26,028 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/83b8d86696ce4c728d410b325dfe4099 is 169, key is trslm:hdfs://localhost:42739/backupUT\x00test-1700357184679/meta:log-roll-map/1700357244920/Put/seqid=0 2023-11-19T01:27:26,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=68 2023-11-19T01:27:26,432 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/83b8d86696ce4c728d410b325dfe4099 2023-11-19T01:27:26,443 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/05f6b05b598842a0a5af3e603f2dfa9f is 452, key is session:backup_1700357236810/session:context/1700357245343/Put/seqid=0 2023-11-19T01:27:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=68 2023-11-19T01:27:26,848 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/05f6b05b598842a0a5af3e603f2dfa9f 2023-11-19T01:27:26,855 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/83b8d86696ce4c728d410b325dfe4099 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/83b8d86696ce4c728d410b325dfe4099 2023-11-19T01:27:26,861 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/83b8d86696ce4c728d410b325dfe4099, entries=4, sequenceid=50, filesize=5.5 K 2023-11-19T01:27:26,862 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/05f6b05b598842a0a5af3e603f2dfa9f as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/05f6b05b598842a0a5af3e603f2dfa9f 2023-11-19T01:27:26,868 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/05f6b05b598842a0a5af3e603f2dfa9f, entries=2, sequenceid=50, filesize=5.4 K 2023-11-19T01:27:26,869 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 322466352cac3ade468ab2ce8e706acc in 846ms, sequenceid=50, compaction requested=true 2023-11-19T01:27:26,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.HRegion(2520): Flush status journal for 322466352cac3ade468ab2ce8e706acc: 2023-11-19T01:27:26,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. for snapshot_backup_system completed. 2023-11-19T01:27:26,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:27:26,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:27:26,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/83b8d86696ce4c728d410b325dfe4099] hfiles 2023-11-19T01:27:26,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (1/4): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757 for snapshot=snapshot_backup_system 2023-11-19T01:27:26,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (2/4): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63 for snapshot=snapshot_backup_system 2023-11-19T01:27:26,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (3/4): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2 for snapshot=snapshot_backup_system 2023-11-19T01:27:26,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (4/4): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/83b8d86696ce4c728d410b325dfe4099 for snapshot=snapshot_backup_system 2023-11-19T01:27:26,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/05f6b05b598842a0a5af3e603f2dfa9f] hfiles 2023-11-19T01:27:26,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99 for snapshot=snapshot_backup_system 2023-11-19T01:27:26,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff for snapshot=snapshot_backup_system 2023-11-19T01:27:26,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460 for snapshot=snapshot_backup_system 2023-11-19T01:27:26,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1 for snapshot=snapshot_backup_system 2023-11-19T01:27:26,869 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/05f6b05b598842a0a5af3e603f2dfa9f for snapshot=snapshot_backup_system 2023-11-19T01:27:27,275 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:27:27,275 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=69 2023-11-19T01:27:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=69 2023-11-19T01:27:27,275 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:27:27,277 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=69, ppid=68, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:27:27,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=69, resume processing ppid=68 2023-11-19T01:27:27,279 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:27:27,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=69, ppid=68, state=SUCCESS; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc in 1.4070 sec 2023-11-19T01:27:27,279 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:27:27,280 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:27:27,280 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:27:27,281 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:27:27,291 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:27:27,302 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:27:27,303 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:27:27,305 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=68, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:27:27,305 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 68 2023-11-19T01:27:27,306 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=68, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=68, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8490 sec 2023-11-19T01:27:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=68 2023-11-19T01:27:27,564 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(115): Backup backup_1700357245418 started at 1700357247564. 2023-11-19T01:27:27,566 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(119): Backup session backup_1700357245418 has been started. 2023-11-19T01:27:27,571 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-19T01:27:27,571 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:27:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-11-19T01:27:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:27:27,572 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:27:27,572 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:27:27,573 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:27:27,573 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:27:27,573 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:27:27,575 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:27,575 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:27:27,575 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:27:27,575 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:27:27,575 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:27:27,576 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:27:27,576 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:27:27,576 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:27:27,576 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:27:27,576 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-11-19T01:27:27,576 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:27:27,576 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:27:27,576 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:27:27,577 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:27:27,577 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:27:27,577 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:27:27,580 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:27,580 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:27,580 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:27:27,580 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:27:27,580 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:27:27,580 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:27:27,581 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:27:27,581 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:27,581 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:27,582 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:27:27,582 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:27:27,582 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@192550e1[Count = 0] remaining members to acquire global barrier 2023-11-19T01:27:27,582 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:27:27,582 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:27,587 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:27,587 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:27,587 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:27:27,587 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:27:27,587 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:27:27,587 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:27,587 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357240980 highest: 1700357240980 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:27,587 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784:(num 1700357240980) roll requested 2023-11-19T01:27:27,599 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357247588, exclude list is [], retry=0 2023-11-19T01:27:27,603 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:27:27,604 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357240980 with entries=10, filesize=2.31 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357247588 2023-11-19T01:27:27,604 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:27:27,604 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357240980 is not closed yet, will try archiving it next time 2023-11-19T01:27:27,605 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta:.meta(num 1700357240997) roll requested 2023-11-19T01:27:27,605 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357231388 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/oldWALs/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357231388 2023-11-19T01:27:27,607 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357240980 not finished, retry = 0 2023-11-19T01:27:27,617 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357247605.meta, exclude list is [], retry=0 2023-11-19T01:27:27,620 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:27:27,621 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357240997.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357247605.meta 2023-11-19T01:27:27,621 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:27:27,621 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357240997.meta is not closed yet, will try archiving it next time 2023-11-19T01:27:27,623 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357240997.meta to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/oldWALs/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357240997.meta 2023-11-19T01:27:27,628 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-11-19T01:27:27,628 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357247588 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:27,636 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:27:27,636 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,44997,1700357176784' in zk 2023-11-19T01:27:27,638 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:27:27,638 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:27,638 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:27:27,638 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:27:27,638 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:27:27,639 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:27:27,639 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:27:27,639 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:27:27,640 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:27,640 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:27,640 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:27:27,641 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:27,641 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:27,642 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,44997,1700357176784': 2023-11-19T01:27:27,642 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:27:27,642 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:27:27,642 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:27:27,642 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:27:27,642 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:27:27,642 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:27:27,644 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:27:27,644 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:27:27,644 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:27:27,644 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:27:27,644 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:27:27,644 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:27:27,644 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:27:27,644 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:27:27,645 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:27:27,645 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:27,645 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:27:27,645 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:27,646 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:27,646 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:27:27,646 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:27,647 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:27,654 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:27:27,654 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:27:27,654 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:27:27,654 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:27:27,655 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:27:27,655 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:27:27,655 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:27:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-19T01:27:27,759 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357247758_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } 2023-11-19T01:27:27,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357247759 (current time:1700357247759). 2023-11-19T01:27:27,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:27:27,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357247758_default_test-1700357184679 VERSION not specified, setting to 2 2023-11-19T01:27:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f602610, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:27,768 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:55696, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:27,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:27:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=70, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1700357247758_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } 2023-11-19T01:27:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357247758_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 }, snapshot procedure id = 70 2023-11-19T01:27:27,774 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1700357247758_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:27:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=70 2023-11-19T01:27:27,775 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1700357247758_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:27:27,777 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357247758_default_test-1700357184679 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:27:27,779 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1700357247758_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:27:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=70 2023-11-19T01:27:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=70 2023-11-19T01:27:28,185 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1700357247758_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:27:28,186 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE; SnapshotRegionProcedure 98c2f2b8120be772f779a6f83df59517}] 2023-11-19T01:27:28,188 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=71, ppid=70, state=RUNNABLE; SnapshotRegionProcedure 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:28,338 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:28,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=71 2023-11-19T01:27:28,339 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. 2023-11-19T01:27:28,339 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.HRegion(2520): Flush status journal for 98c2f2b8120be772f779a6f83df59517: 2023-11-19T01:27:28,339 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. for snapshot_1700357247758_default_test-1700357184679 completed. 2023-11-19T01:27:28,340 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(240): Storing 'test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517.' region-info for snapshot=snapshot_1700357247758_default_test-1700357184679 2023-11-19T01:27:28,340 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:27:28,340 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_] hfiles 2023-11-19T01:27:28,340 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ for snapshot=snapshot_1700357247758_default_test-1700357184679 2023-11-19T01:27:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=70 2023-11-19T01:27:28,745 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. 2023-11-19T01:27:28,745 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=71 2023-11-19T01:27:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=71 2023-11-19T01:27:28,746 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357247758_default_test-1700357184679 on region 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:28,747 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=71, ppid=70, state=RUNNABLE; SnapshotRegionProcedure 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:28,749 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=71, resume processing ppid=70 2023-11-19T01:27:28,749 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=71, ppid=70, state=SUCCESS; SnapshotRegionProcedure 98c2f2b8120be772f779a6f83df59517 in 562 msec 2023-11-19T01:27:28,749 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1700357247758_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:27:28,750 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1700357247758_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:27:28,750 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1700357247758_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:27:28,750 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357247758_default_test-1700357184679 2023-11-19T01:27:28,751 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357247758_default_test-1700357184679 2023-11-19T01:27:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=70 2023-11-19T01:27:29,160 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1700357247758_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:27:29,166 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1700357247758_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:27:29,167 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357247758_default_test-1700357184679 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357247758_default_test-1700357184679 2023-11-19T01:27:29,168 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=70, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1700357247758_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:27:29,168 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357247758_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 }, snapshot procedure id = 70 2023-11-19T01:27:29,169 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=70, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=70, snapshot={ ss=snapshot_1700357247758_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } in 1.3960 sec 2023-11-19T01:27:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=70 2023-11-19T01:27:29,880 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1700357245418 2023-11-19T01:27:29,880 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-19T01:27:29,880 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-19T01:27:29,880 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357245418_test-1700357184679 2023-11-19T01:27:29,880 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357247758_default_test-1700357184679 to hdfs://localhost:42739/backupUT/backup_1700357245418/default/test-1700357184679/ 2023-11-19T01:27:29,880 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:27:29,904 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:27:29,904 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,44889,1700357173572_-1120179319_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42739/backupUT/backup_1700357245418/default/test-1700357184679, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42739/backupUT/backup_1700357245418/default/test-1700357184679/.hbase-snapshot/.tmp/snapshot_1700357247758_default_test-1700357184679 2023-11-19T01:27:29,904 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af, snapshotDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357247758_default_test-1700357184679. 2023-11-19T01:27:29,912 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357247758_default_test-1700357184679 to hdfs://localhost:42739/backupUT/backup_1700357245418/default/test-1700357184679/.hbase-snapshot/.tmp/snapshot_1700357247758_default_test-1700357184679 2023-11-19T01:27:30,336 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:30,336 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:30,337 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:30,337 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:31,354 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-6984200308958940774.jar 2023-11-19T01:27:31,354 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:31,355 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:31,355 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:31,356 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:31,356 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:31,356 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:31,357 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:31,357 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:31,357 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:31,358 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:27:31,359 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:27:31,359 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:27:31,359 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:27:31,360 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:27:31,360 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:27:31,361 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:27:31,361 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:27:31,362 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:27:31,362 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:27:31,362 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:27:31,363 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:27:31,363 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:27:31,364 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:27:31,365 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:27:31,365 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:27:31,365 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:27:31,366 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:27:31,366 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:27:31,417 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357247758_default_test-1700357184679' hfile list 2023-11-19T01:27:31,419 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-19T01:27:31,488 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357251455/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:27:31,489 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357251455/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:27:31,542 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-19T01:27:31,567 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ output=hdfs://localhost:42739/backupUT/backup_1700357245418/default/test-1700357184679/archive/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ 2023-11-19T01:27:31,567 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 4.160M/sec 2023-11-19T01:27:32,521 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:27:32,522 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:27:32,529 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357247758_default_test-1700357184679 2023-11-19T01:27:32,529 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357247758_default_test-1700357184679 finished. 2023-11-19T01:27:32,529 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1034): test-1700357184679 2023-11-19T01:27:32,537 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357245418 2023-11-19T01:27:32,537 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:27:32,542 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42739/backupUT/backup_1700357245418/.backup.manifest 2023-11-19T01:27:32,542 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(358): Backup backup_1700357245418 finished: type=FULL,tablelist=test-1700357184679,targetRootDir=hdfs://localhost:42739/backupUT,startts=1700357247564,completets=1700357252537,bytescopied=0 2023-11-19T01:27:32,542 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-19T01:27:32,542 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357247758_default_test-1700357184679 2023-11-19T01:27:32,543 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357247758_default_test-1700357184679" 2023-11-19T01:27:32,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357247758_default_test-1700357184679 2023-11-19T01:27:32,547 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357247758_default_test-1700357184679 for backup backup_1700357245418 succeeded. 2023-11-19T01:27:32,548 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:27:32,553 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:27:32,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:27:32,556 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:27:32,558 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:27:32,563 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(376): Backup backup_1700357245418 completed. 2023-11-19T01:27:32,563 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:32,582 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestFullRestore(61): backup complete 2023-11-19T01:27:32,583 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-11-19T01:27:32,584 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357245418/.backup.manifest 2023-11-19T01:27:32,586 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357245418/.backup.manifest 2023-11-19T01:27:32,587 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357245418/.backup.manifest 2023-11-19T01:27:32,589 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357245418/.backup.manifest 2023-11-19T01:27:32,589 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(148): Restoring 'test-1700357184679' to 'table1' from full backup image hdfs://localhost:42739/backupUT/backup_1700357245418/default/test-1700357184679 2023-11-19T01:27:32,597 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(487): Creating target table 'table1' 2023-11-19T01:27:32,597 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:42739/backupUT/backup_1700357245418/default/test-1700357184679/archive/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:32,598 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:42739/backupUT/backup_1700357245418/default/test-1700357184679/archive/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f in region [hdfs://localhost:42739/backupUT/backup_1700357245418/default/test-1700357184679/archive/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517] 2023-11-19T01:27:32,602 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:42739/backupUT/backup_1700357245418/default/test-1700357184679/archive/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ first=row0 last=row98 2023-11-19T01:27:32,604 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:27:32,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-11-19T01:27:32,607 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:27:32,607 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 72 2023-11-19T01:27:32,607 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:27:32,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=72 2023-11-19T01:27:32,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=72 2023-11-19T01:27:32,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=72 2023-11-19T01:27:33,015 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 1090f6032ec6122583f9a8126b411178, NAME => 'table1,,1700357252603.1090f6032ec6122583f9a8126b411178.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:27:33,021 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1700357252603.1090f6032ec6122583f9a8126b411178.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:27:33,021 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing 1090f6032ec6122583f9a8126b411178, disabling compactions & flushes 2023-11-19T01:27:33,021 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1700357252603.1090f6032ec6122583f9a8126b411178. 2023-11-19T01:27:33,021 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1700357252603.1090f6032ec6122583f9a8126b411178. 2023-11-19T01:27:33,021 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1700357252603.1090f6032ec6122583f9a8126b411178. after waiting 0 ms 2023-11-19T01:27:33,021 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1700357252603.1090f6032ec6122583f9a8126b411178. 2023-11-19T01:27:33,021 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1700357252603.1090f6032ec6122583f9a8126b411178. 2023-11-19T01:27:33,021 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for 1090f6032ec6122583f9a8126b411178: 2023-11-19T01:27:33,022 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:27:33,022 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1700357252603.1090f6032ec6122583f9a8126b411178.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1700357253022"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357253022"}]},"ts":"1700357253022"} 2023-11-19T01:27:33,024 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:27:33,025 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:27:33,025 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357253025"}]},"ts":"1700357253025"} 2023-11-19T01:27:33,026 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-11-19T01:27:33,032 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=1090f6032ec6122583f9a8126b411178, ASSIGN}] 2023-11-19T01:27:33,034 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=1090f6032ec6122583f9a8126b411178, ASSIGN 2023-11-19T01:27:33,035 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=73, ppid=72, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=1090f6032ec6122583f9a8126b411178, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,44997,1700357176784; forceNewPlan=false, retain=false 2023-11-19T01:27:33,187 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=1090f6032ec6122583f9a8126b411178, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:33,189 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; OpenRegionProcedure 1090f6032ec6122583f9a8126b411178, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:27:33,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=72 2023-11-19T01:27:33,340 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:33,343 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(130): Open table1,,1700357252603.1090f6032ec6122583f9a8126b411178. 2023-11-19T01:27:33,343 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7254): Opening region: {ENCODED => 1090f6032ec6122583f9a8126b411178, NAME => 'table1,,1700357252603.1090f6032ec6122583f9a8126b411178.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:27:33,344 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:27:33,344 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 1090f6032ec6122583f9a8126b411178 2023-11-19T01:27:33,344 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(888): Instantiated table1,,1700357252603.1090f6032ec6122583f9a8126b411178.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:27:33,344 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7296): checking encryption for 1090f6032ec6122583f9a8126b411178 2023-11-19T01:27:33,344 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7299): checking classloading for 1090f6032ec6122583f9a8126b411178 2023-11-19T01:27:33,345 INFO [StoreOpener-1090f6032ec6122583f9a8126b411178-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 1090f6032ec6122583f9a8126b411178 2023-11-19T01:27:33,347 INFO [StoreOpener-1090f6032ec6122583f9a8126b411178-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1090f6032ec6122583f9a8126b411178 columnFamilyName f 2023-11-19T01:27:33,347 DEBUG [StoreOpener-1090f6032ec6122583f9a8126b411178-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:27:33,347 INFO [StoreOpener-1090f6032ec6122583f9a8126b411178-1 {}] regionserver.HStore(324): Store=1090f6032ec6122583f9a8126b411178/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:27:33,348 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/1090f6032ec6122583f9a8126b411178 2023-11-19T01:27:33,348 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/1090f6032ec6122583f9a8126b411178 2023-11-19T01:27:33,350 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1076): writing seq id for 1090f6032ec6122583f9a8126b411178 2023-11-19T01:27:33,352 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/1090f6032ec6122583f9a8126b411178/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:27:33,353 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): Opened 1090f6032ec6122583f9a8126b411178; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10561080000, jitterRate=-0.016422778367996216}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:27:33,354 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(992): Region open journal for 1090f6032ec6122583f9a8126b411178: 2023-11-19T01:27:33,355 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2556): Post open deploy tasks for table1,,1700357252603.1090f6032ec6122583f9a8126b411178., pid=74, masterSystemTime=1700357253340 2023-11-19T01:27:33,356 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2583): Finished post open deploy task for table1,,1700357252603.1090f6032ec6122583f9a8126b411178. 2023-11-19T01:27:33,356 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(158): Opened table1,,1700357252603.1090f6032ec6122583f9a8126b411178. 2023-11-19T01:27:33,356 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=73 updating hbase:meta row=1090f6032ec6122583f9a8126b411178, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:33,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=74, resume processing ppid=73 2023-11-19T01:27:33,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=74, ppid=73, state=SUCCESS; OpenRegionProcedure 1090f6032ec6122583f9a8126b411178, server=jenkins-hbase6.apache.org,44997,1700357176784 in 169 msec 2023-11-19T01:27:33,360 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=73, resume processing ppid=72 2023-11-19T01:27:33,361 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=73, ppid=72, state=SUCCESS; TransitRegionStateProcedure table=table1, region=1090f6032ec6122583f9a8126b411178, ASSIGN in 327 msec 2023-11-19T01:27:33,361 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:27:33,361 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357253361"}]},"ts":"1700357253361"} 2023-11-19T01:27:33,362 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-11-19T01:27:33,364 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=72, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:27:33,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=72, state=SUCCESS; CreateTableProcedure table=table1 in 760 msec 2023-11-19T01:27:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=72 2023-11-19T01:27:33,715 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:42739/backupUT/backup_1700357245418/default/test-1700357184679/archive/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517 from hbase tables test-1700357184679 to tables table1 2023-11-19T01:27:33,715 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1700357184679 into table1 2023-11-19T01:27:33,717 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-table1-1700357253715 from hdfs://localhost:42739/backupUT/backup_1700357245418/default/test-1700357184679/archive/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:33,723 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b6b67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:33,724 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:27:33,725 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:45584, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:27:33,726 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-19T01:27:33,726 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-11-19T01:27:33,732 DEBUG [hconnection-0x1e74bc30-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:33,734 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:42864, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:33,736 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-11-19T01:27:33,736 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-11-19T01:27:33,736 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/partitions_5899a2d5-9fcc-4f8e-8a57-d7a32f5b3530 2023-11-19T01:27:34,145 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:34,146 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:34,146 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:34,147 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:34,784 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-11-19T01:27:34,784 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_table1 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:27:35,152 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-2216572544312619025.jar 2023-11-19T01:27:35,153 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:35,153 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:35,153 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:35,154 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:35,154 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:35,154 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:35,155 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:35,155 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:35,155 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:35,156 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:27:35,157 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:27:35,157 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:27:35,158 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:27:35,158 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:27:35,159 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:27:35,159 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:27:35,159 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:27:35,160 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:27:35,160 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:27:35,161 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:27:35,161 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:27:35,161 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:35,162 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:35,162 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:35,162 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:35,163 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:35,163 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:35,164 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:27:35,164 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-11-19T01:27:35,164 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:35,164 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-11-19T01:27:35,164 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-11-19T01:27:35,165 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:27:35,178 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-11-19T01:27:35,257 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357255221/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:27:35,257 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357255221/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:27:35,307 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:42739/backupUT/backup_1700357245418/default/test-1700357184679/archive/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ 2023-11-19T01:27:35,312 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-11-19T01:27:35,333 DEBUG [pool-300-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eb84c3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:35,335 DEBUG [hconnection-0x7e3425a0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:35,336 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:42870, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:35,338 DEBUG [pool-300-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:35,339 DEBUG [pool-300-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-11-19T01:27:35,339 DEBUG [pool-300-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase6.apache.org 2023-11-19T01:27:35,348 DEBUG [pool-300-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-table1-1700357253715/_temporary/0/_temporary/attempt_local1712796440_0009_r_000000_0/f/59ffa6b2e0354a3783ee8f56ad72715c is 37, key is row10/f:q1/1700357187320/Put/seqid=0 2023-11-19T01:27:35,350 WARN [Thread-1257 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase6.apache.org:44997] Specified favored nodes: [jenkins-hbase6.apache.org:44997] 2023-11-19T01:27:36,303 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-table1-1700357253715 2023-11-19T01:27:36,309 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b603200, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:36,310 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:36,311 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:42880, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:36,317 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-table1-1700357253715/_SUCCESS 2023-11-19T01:27:36,321 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:27:36,322 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:45596, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:27:36,334 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-table1-1700357253715/f/59ffa6b2e0354a3783ee8f56ad72715c first=Optional[row0] last=Optional[row98] 2023-11-19T01:27:36,334 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1700357252603.1090f6032ec6122583f9a8126b411178., hostname=jenkins-hbase6.apache.org,44997,1700357176784, seqNum=2 for row with hfile group [{f,hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-table1-1700357253715/f/59ffa6b2e0354a3783ee8f56ad72715c}] 2023-11-19T01:27:36,340 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-table1-1700357253715/f/59ffa6b2e0354a3783ee8f56ad72715c for inclusion in 1090f6032ec6122583f9a8126b411178/f 2023-11-19T01:27:36,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-11-19T01:27:36,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(625): Region bounds: first= last= 2023-11-19T01:27:36,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegion(2520): Flush status journal for 1090f6032ec6122583f9a8126b411178: 2023-11-19T01:27:36,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-table1-1700357253715/f/59ffa6b2e0354a3783ee8f56ad72715c to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__table1__7o83ml1f0ejnu93coptt7iovf8p5nf0cjfdl8rij4n6qi4oi1tg44s0l5g44pei0/f/59ffa6b2e0354a3783ee8f56ad72715c 2023-11-19T01:27:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__table1__7o83ml1f0ejnu93coptt7iovf8p5nf0cjfdl8rij4n6qi4oi1tg44s0l5g44pei0/f/59ffa6b2e0354a3783ee8f56ad72715c as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/1090f6032ec6122583f9a8126b411178/f/a112a67e4b934d37be5007698ea9ddaa_SeqId_4_ 2023-11-19T01:27:36,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9f35dab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:36,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:27:36,357 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:45606, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:27:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:36,361 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:42882, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:36,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:36,373 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__table1__7o83ml1f0ejnu93coptt7iovf8p5nf0cjfdl8rij4n6qi4oi1tg44s0l5g44pei0/f/59ffa6b2e0354a3783ee8f56ad72715c into 1090f6032ec6122583f9a8126b411178/f as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/1090f6032ec6122583f9a8126b411178/f/a112a67e4b934d37be5007698ea9ddaa_SeqId_4_ - updating store file list. 2023-11-19T01:27:36,379 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/1090f6032ec6122583f9a8126b411178/f/a112a67e4b934d37be5007698ea9ddaa_SeqId_4_ into 1090f6032ec6122583f9a8126b411178/f 2023-11-19T01:27:36,379 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__table1__7o83ml1f0ejnu93coptt7iovf8p5nf0cjfdl8rij4n6qi4oi1tg44s0l5g44pei0/f/59ffa6b2e0354a3783ee8f56ad72715c into 1090f6032ec6122583f9a8126b411178/f (new location: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/1090f6032ec6122583f9a8126b411178/f/a112a67e4b934d37be5007698ea9ddaa_SeqId_4_) 2023-11-19T01:27:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__table1__7o83ml1f0ejnu93coptt7iovf8p5nf0cjfdl8rij4n6qi4oi1tg44s0l5g44pei0/f/59ffa6b2e0354a3783ee8f56ad72715c 2023-11-19T01:27:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c32edf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:27:36,386 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:45608, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-19T01:27:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:36,390 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:42888, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-11-19T01:27:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:36,403 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:36,403 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-11-19T01:27:36,403 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-19T01:27:36,403 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(233): Backup: backup_1700357245418 hdfs://localhost:42739/backupUT/backup_1700357245418/default/test-1700357184679/ 2023-11-19T01:27:36,403 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-11-19T01:27:36,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable table1 2023-11-19T01:27:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=75, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-11-19T01:27:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=75 2023-11-19T01:27:36,409 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357256409"}]},"ts":"1700357256409"} 2023-11-19T01:27:36,410 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-11-19T01:27:36,412 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-11-19T01:27:36,412 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=1090f6032ec6122583f9a8126b411178, UNASSIGN}] 2023-11-19T01:27:36,414 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=1090f6032ec6122583f9a8126b411178, UNASSIGN 2023-11-19T01:27:36,414 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=1090f6032ec6122583f9a8126b411178, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:36,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE; CloseRegionProcedure 1090f6032ec6122583f9a8126b411178, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:27:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=75 2023-11-19T01:27:36,567 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:36,567 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(119): Close 1090f6032ec6122583f9a8126b411178 2023-11-19T01:27:36,567 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1668): Closing 1090f6032ec6122583f9a8126b411178, disabling compactions & flushes 2023-11-19T01:27:36,567 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1690): Closing region table1,,1700357252603.1090f6032ec6122583f9a8126b411178. 2023-11-19T01:27:36,567 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1700357252603.1090f6032ec6122583f9a8126b411178. 2023-11-19T01:27:36,567 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1778): Acquired close lock on table1,,1700357252603.1090f6032ec6122583f9a8126b411178. after waiting 0 ms 2023-11-19T01:27:36,567 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1788): Updates disabled for region table1,,1700357252603.1090f6032ec6122583f9a8126b411178. 2023-11-19T01:27:36,574 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/1090f6032ec6122583f9a8126b411178/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-11-19T01:27:36,574 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:27:36,575 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1909): Closed table1,,1700357252603.1090f6032ec6122583f9a8126b411178. 2023-11-19T01:27:36,575 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] regionserver.HRegion(1622): Region close journal for 1090f6032ec6122583f9a8126b411178: 2023-11-19T01:27:36,576 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=77}] handler.UnassignRegionHandler(163): Closed 1090f6032ec6122583f9a8126b411178 2023-11-19T01:27:36,576 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=76 updating hbase:meta row=1090f6032ec6122583f9a8126b411178, regionState=CLOSED 2023-11-19T01:27:36,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=77, resume processing ppid=76 2023-11-19T01:27:36,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=77, ppid=76, state=SUCCESS; CloseRegionProcedure 1090f6032ec6122583f9a8126b411178, server=jenkins-hbase6.apache.org,44997,1700357176784 in 163 msec 2023-11-19T01:27:36,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=76, resume processing ppid=75 2023-11-19T01:27:36,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=76, ppid=75, state=SUCCESS; TransitRegionStateProcedure table=table1, region=1090f6032ec6122583f9a8126b411178, UNASSIGN in 167 msec 2023-11-19T01:27:36,581 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357256581"}]},"ts":"1700357256581"} 2023-11-19T01:27:36,582 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-11-19T01:27:36,584 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-11-19T01:27:36,586 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=75, state=SUCCESS; DisableTableProcedure table=table1 in 180 msec 2023-11-19T01:27:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=75 2023-11-19T01:27:36,717 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$5(2449): Client=jenkins//172.31.2.20 delete table1 2023-11-19T01:27:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-11-19T01:27:36,730 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=78, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-11-19T01:27:36,731 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=78, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-11-19T01:27:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=78 2023-11-19T01:27:36,734 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/1090f6032ec6122583f9a8126b411178 2023-11-19T01:27:36,738 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/1090f6032ec6122583f9a8126b411178/f, FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/1090f6032ec6122583f9a8126b411178/recovered.edits] 2023-11-19T01:27:36,741 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/1090f6032ec6122583f9a8126b411178/f/a112a67e4b934d37be5007698ea9ddaa_SeqId_4_ to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/archive/data/default/table1/1090f6032ec6122583f9a8126b411178/f/a112a67e4b934d37be5007698ea9ddaa_SeqId_4_ 2023-11-19T01:27:36,747 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/1090f6032ec6122583f9a8126b411178/recovered.edits/6.seqid to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/archive/data/default/table1/1090f6032ec6122583f9a8126b411178/recovered.edits/6.seqid 2023-11-19T01:27:36,747 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/1090f6032ec6122583f9a8126b411178 2023-11-19T01:27:36,747 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-11-19T01:27:36,749 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=78, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-11-19T01:27:36,751 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-11-19T01:27:36,753 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-11-19T01:27:36,754 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=78, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-11-19T01:27:36,754 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-11-19T01:27:36,754 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1700357252603.1090f6032ec6122583f9a8126b411178.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357256754"}]},"ts":"9223372036854775807"} 2023-11-19T01:27:36,755 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:27:36,755 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 1090f6032ec6122583f9a8126b411178, NAME => 'table1,,1700357252603.1090f6032ec6122583f9a8126b411178.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:27:36,755 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-11-19T01:27:36,756 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1700357256755"}]},"ts":"9223372036854775807"} 2023-11-19T01:27:36,757 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-11-19T01:27:36,760 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(127): Finished pid=78, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-11-19T01:27:36,763 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=78, state=SUCCESS; DeleteTableProcedure table=table1 in 42 msec 2023-11-19T01:27:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=78 2023-11-19T01:27:36,851 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingle Thread=596 (was 596), OpenFileDescriptor=710 (was 709) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=149 (was 162), ProcessCount=166 (was 166), AvailableMemoryMB=3667 (was 3770) 2023-11-19T01:27:36,851 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=596 is superior to 500 2023-11-19T01:27:36,866 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleOverwriteCommand Thread=596, OpenFileDescriptor=710, MaxFileDescriptor=60000, SystemLoadAverage=149, ProcessCount=166, AvailableMemoryMB=3666 2023-11-19T01:27:36,866 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=596 is superior to 500 2023-11-19T01:27:36,867 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestFullRestore(192): test full restore on a single table empty table: command-line 2023-11-19T01:27:36,873 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cb10ece, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:36,875 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:36,877 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:42892, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:36,878 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:27:36,880 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:45624, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:27:36,890 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:27:36,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:27:36,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357256894 (current time:1700357256894). 2023-11-19T01:27:36,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:27:36,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:27:36,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67612c5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:36,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:36,902 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:42898, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:36,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:36,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:27:36,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=79, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:27:36,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-11-19T01:27:36,906 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:27:36,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-19T01:27:36,907 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:27:36,910 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:27:36,913 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:27:37,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-19T01:27:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-19T01:27:37,319 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:27:37,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc}] 2023-11-19T01:27:37,321 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=80, ppid=79, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:27:37,472 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:37,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=80 2023-11-19T01:27:37,473 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:27:37,473 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegion(2819): Flushing 322466352cac3ade468ab2ce8e706acc 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-11-19T01:27:37,477 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/817bbb8696494171be63b200e480be74 is 169, key is trslm:hdfs://localhost:42739/backupUT\x00test-1700357184679/meta:log-roll-map/1700357252532/Put/seqid=0 2023-11-19T01:27:37,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-19T01:27:37,881 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/817bbb8696494171be63b200e480be74 2023-11-19T01:27:37,890 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/6e39103009374e778431dfd7316f5a4c is 452, key is session:backup_1700357245418/session:context/1700357252557/Put/seqid=0 2023-11-19T01:27:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-19T01:27:38,294 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/6e39103009374e778431dfd7316f5a4c 2023-11-19T01:27:38,300 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/817bbb8696494171be63b200e480be74 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/817bbb8696494171be63b200e480be74 2023-11-19T01:27:38,305 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/817bbb8696494171be63b200e480be74, entries=4, sequenceid=61, filesize=5.5 K 2023-11-19T01:27:38,306 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/6e39103009374e778431dfd7316f5a4c as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/6e39103009374e778431dfd7316f5a4c 2023-11-19T01:27:38,310 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/6e39103009374e778431dfd7316f5a4c, entries=2, sequenceid=61, filesize=5.4 K 2023-11-19T01:27:38,311 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 322466352cac3ade468ab2ce8e706acc in 838ms, sequenceid=61, compaction requested=true 2023-11-19T01:27:38,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.HRegion(2520): Flush status journal for 322466352cac3ade468ab2ce8e706acc: 2023-11-19T01:27:38,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. for snapshot_backup_system completed. 2023-11-19T01:27:38,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:27:38,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:27:38,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/83b8d86696ce4c728d410b325dfe4099, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/817bbb8696494171be63b200e480be74] hfiles 2023-11-19T01:27:38,311 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (1/5): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757 for snapshot=snapshot_backup_system 2023-11-19T01:27:38,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (2/5): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63 for snapshot=snapshot_backup_system 2023-11-19T01:27:38,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (3/5): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2 for snapshot=snapshot_backup_system 2023-11-19T01:27:38,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (4/5): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/83b8d86696ce4c728d410b325dfe4099 for snapshot=snapshot_backup_system 2023-11-19T01:27:38,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (5/5): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/817bbb8696494171be63b200e480be74 for snapshot=snapshot_backup_system 2023-11-19T01:27:38,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/05f6b05b598842a0a5af3e603f2dfa9f, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/6e39103009374e778431dfd7316f5a4c] hfiles 2023-11-19T01:27:38,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99 for snapshot=snapshot_backup_system 2023-11-19T01:27:38,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff for snapshot=snapshot_backup_system 2023-11-19T01:27:38,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460 for snapshot=snapshot_backup_system 2023-11-19T01:27:38,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1 for snapshot=snapshot_backup_system 2023-11-19T01:27:38,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/05f6b05b598842a0a5af3e603f2dfa9f for snapshot=snapshot_backup_system 2023-11-19T01:27:38,312 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/6e39103009374e778431dfd7316f5a4c for snapshot=snapshot_backup_system 2023-11-19T01:27:38,717 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:27:38,717 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=80 2023-11-19T01:27:38,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=80 2023-11-19T01:27:38,717 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:27:38,719 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=80, ppid=79, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:27:38,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=80, resume processing ppid=79 2023-11-19T01:27:38,721 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:27:38,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=80, ppid=79, state=SUCCESS; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc in 1.3990 sec 2023-11-19T01:27:38,721 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:27:38,722 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:27:38,722 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:27:38,723 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:27:38,887 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:27:39,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-19T01:27:39,143 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:27:39,153 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:27:39,153 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:27:39,154 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=79, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:27:39,154 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 79 2023-11-19T01:27:39,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=79, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2500 sec 2023-11-19T01:27:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=79 2023-11-19T01:27:41,013 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(115): Backup backup_1700357256873 started at 1700357261013. 2023-11-19T01:27:41,015 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(119): Backup session backup_1700357256873 has been started. 2023-11-19T01:27:41,020 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-19T01:27:41,020 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:27:41,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:27:41,020 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:27:41,021 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:27:41,022 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:27:41,022 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:27:41,022 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:27:41,024 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:41,024 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:27:41,024 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:27:41,024 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:27:41,024 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:27:41,025 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:27:41,025 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:27:41,025 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:27:41,025 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:27:41,025 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:27:41,026 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:27:41,026 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:27:41,026 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:27:41,026 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:27:41,026 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:27:41,030 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:41,030 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:41,030 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:27:41,030 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:27:41,030 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:27:41,030 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:27:41,031 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:27:41,031 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:41,031 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:41,032 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:27:41,032 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:27:41,032 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@79bdabf0[Count = 0] remaining members to acquire global barrier 2023-11-19T01:27:41,032 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:27:41,032 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:41,033 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:41,034 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:41,034 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:27:41,034 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:27:41,034 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:41,034 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:27:41,034 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357247588 highest: 1700357247588 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:41,034 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784:(num 1700357247588) roll requested 2023-11-19T01:27:41,045 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357261034, exclude list is [], retry=0 2023-11-19T01:27:41,050 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:27:41,052 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357247588 with entries=13, filesize=3.09 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357261034 2023-11-19T01:27:41,052 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:27:41,052 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357247588 is not closed yet, will try archiving it next time 2023-11-19T01:27:41,052 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta:.meta(num 1700357247605) roll requested 2023-11-19T01:27:41,052 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357240980 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/oldWALs/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357240980 2023-11-19T01:27:41,063 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357261052.meta, exclude list is [], retry=0 2023-11-19T01:27:41,066 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:27:41,067 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357247605.meta with entries=12, filesize=2.73 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357261052.meta 2023-11-19T01:27:41,068 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:27:41,068 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357247605.meta is not closed yet, will try archiving it next time 2023-11-19T01:27:41,074 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-19T01:27:41,074 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357261034 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:41,082 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:27:41,082 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,44997,1700357176784' in zk 2023-11-19T01:27:41,084 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:27:41,084 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:41,084 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:27:41,084 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:27:41,084 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:27:41,084 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:27:41,085 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:27:41,086 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:27:41,086 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:41,086 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:41,087 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:27:41,087 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:41,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:41,088 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,44997,1700357176784': 2023-11-19T01:27:41,088 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:27:41,088 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:27:41,088 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:27:41,088 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:27:41,088 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:27:41,088 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:27:41,090 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:27:41,090 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:27:41,090 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:27:41,090 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:27:41,090 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:27:41,091 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:27:41,091 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:27:41,091 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:27:41,091 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:27:41,091 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:41,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:27:41,092 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:41,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:41,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:27:41,093 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:41,094 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:41,098 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:27:41,098 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:27:41,099 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:27:41,099 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:27:41,099 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:27:41,099 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:27:41,099 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:27:41,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-19T01:27:41,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357261202_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } 2023-11-19T01:27:41,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357261203 (current time:1700357261203). 2023-11-19T01:27:41,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:27:41,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357261202_default_test-1700357184679 VERSION not specified, setting to 2 2023-11-19T01:27:41,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@712f960a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:41,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:41,212 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:40932, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:41,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:41,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:27:41,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=81, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1700357261202_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } 2023-11-19T01:27:41,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357261202_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 }, snapshot procedure id = 81 2023-11-19T01:27:41,216 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1700357261202_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:27:41,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2023-11-19T01:27:41,217 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1700357261202_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:27:41,219 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357261202_default_test-1700357184679 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:27:41,221 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1700357261202_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:27:41,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2023-11-19T01:27:41,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2023-11-19T01:27:41,627 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1700357261202_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:27:41,627 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE; SnapshotRegionProcedure 98c2f2b8120be772f779a6f83df59517}] 2023-11-19T01:27:41,628 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=82, ppid=81, state=RUNNABLE; SnapshotRegionProcedure 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:41,779 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:41,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=82 2023-11-19T01:27:41,780 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. 2023-11-19T01:27:41,780 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.HRegion(2520): Flush status journal for 98c2f2b8120be772f779a6f83df59517: 2023-11-19T01:27:41,780 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. for snapshot_1700357261202_default_test-1700357184679 completed. 2023-11-19T01:27:41,780 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(240): Storing 'test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517.' region-info for snapshot=snapshot_1700357261202_default_test-1700357184679 2023-11-19T01:27:41,780 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:27:41,780 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_] hfiles 2023-11-19T01:27:41,780 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ for snapshot=snapshot_1700357261202_default_test-1700357184679 2023-11-19T01:27:41,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2023-11-19T01:27:42,186 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. 2023-11-19T01:27:42,186 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=82 2023-11-19T01:27:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=82 2023-11-19T01:27:42,187 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357261202_default_test-1700357184679 on region 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:42,188 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=82, ppid=81, state=RUNNABLE; SnapshotRegionProcedure 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:42,190 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=82, resume processing ppid=81 2023-11-19T01:27:42,190 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=82, ppid=81, state=SUCCESS; SnapshotRegionProcedure 98c2f2b8120be772f779a6f83df59517 in 562 msec 2023-11-19T01:27:42,190 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1700357261202_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:27:42,190 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1700357261202_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:27:42,191 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1700357261202_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:27:42,191 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357261202_default_test-1700357184679 2023-11-19T01:27:42,192 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357261202_default_test-1700357184679 2023-11-19T01:27:42,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2023-11-19T01:27:42,601 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1700357261202_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:27:42,606 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1700357261202_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:27:42,606 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357261202_default_test-1700357184679 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357261202_default_test-1700357184679 2023-11-19T01:27:42,608 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=81, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1700357261202_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:27:42,608 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357261202_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 }, snapshot procedure id = 81 2023-11-19T01:27:42,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=81, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=81, snapshot={ ss=snapshot_1700357261202_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } in 1.3950 sec 2023-11-19T01:27:43,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=81 2023-11-19T01:27:43,322 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1700357256873 2023-11-19T01:27:43,322 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-19T01:27:43,322 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-19T01:27:43,322 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357256873_test-1700357184679 2023-11-19T01:27:43,322 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357261202_default_test-1700357184679 to hdfs://localhost:42739/backupUT/backup_1700357256873/default/test-1700357184679/ 2023-11-19T01:27:43,322 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:27:43,346 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:27:43,346 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,44889,1700357173572_1747425571_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42739/backupUT/backup_1700357256873/default/test-1700357184679, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42739/backupUT/backup_1700357256873/default/test-1700357184679/.hbase-snapshot/.tmp/snapshot_1700357261202_default_test-1700357184679 2023-11-19T01:27:43,346 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af, snapshotDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357261202_default_test-1700357184679. 2023-11-19T01:27:43,354 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357261202_default_test-1700357184679 to hdfs://localhost:42739/backupUT/backup_1700357256873/default/test-1700357184679/.hbase-snapshot/.tmp/snapshot_1700357261202_default_test-1700357184679 2023-11-19T01:27:43,771 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:43,771 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:43,772 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:43,772 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:44,757 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-7666570786169060988.jar 2023-11-19T01:27:44,757 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:44,758 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:44,758 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:44,759 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:44,759 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:44,759 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:44,760 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:44,760 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:44,760 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:44,761 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:27:44,762 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:27:44,762 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:27:44,762 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:27:44,763 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:27:44,763 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:27:44,764 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:27:44,764 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:27:44,764 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:27:44,765 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:27:44,765 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:27:44,766 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:27:44,766 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:27:44,767 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:27:44,767 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:27:44,768 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:27:44,768 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:27:44,768 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:27:44,769 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:27:44,784 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_table1 2023-11-19T01:27:44,812 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357261202_default_test-1700357184679' hfile list 2023-11-19T01:27:44,814 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-19T01:27:44,883 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357264849/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:27:44,883 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357264849/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:27:44,956 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-19T01:27:44,980 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ output=hdfs://localhost:42739/backupUT/backup_1700357256873/default/test-1700357184679/archive/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ 2023-11-19T01:27:44,980 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-11-19T01:27:45,934 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:27:45,935 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:27:45,942 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357261202_default_test-1700357184679 2023-11-19T01:27:45,942 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357261202_default_test-1700357184679 finished. 2023-11-19T01:27:45,942 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1034): test-1700357184679 2023-11-19T01:27:45,948 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357256873 2023-11-19T01:27:45,948 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:27:46,353 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42739/backupUT/backup_1700357256873/.backup.manifest 2023-11-19T01:27:46,353 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(358): Backup backup_1700357256873 finished: type=FULL,tablelist=test-1700357184679,targetRootDir=hdfs://localhost:42739/backupUT,startts=1700357261013,completets=1700357265948,bytescopied=0 2023-11-19T01:27:46,353 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-19T01:27:46,353 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357261202_default_test-1700357184679 2023-11-19T01:27:46,354 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357261202_default_test-1700357184679" 2023-11-19T01:27:46,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357261202_default_test-1700357184679 2023-11-19T01:27:46,357 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357261202_default_test-1700357184679 for backup backup_1700357256873 succeeded. 2023-11-19T01:27:46,358 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:27:46,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:27:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:27:46,365 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:27:46,367 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:27:46,372 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(376): Backup backup_1700357256873 completed. 2023-11-19T01:27:46,372 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:46,383 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestFullRestore(196): backup complete 2023-11-19T01:27:46,383 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] backup.RestoreDriver(101): Found -overwrite option in restore command, will overwrite to existing table if any in the restore target 2023-11-19T01:27:46,389 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5db324fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:46,390 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:46,394 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:40944, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:46,396 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357256873/.backup.manifest 2023-11-19T01:27:46,398 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357256873/.backup.manifest 2023-11-19T01:27:46,399 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357256873/.backup.manifest 2023-11-19T01:27:46,401 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357256873/.backup.manifest 2023-11-19T01:27:46,401 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(148): Restoring 'test-1700357184679' to 'test-1700357184679' from full backup image hdfs://localhost:42739/backupUT/backup_1700357256873/default/test-1700357184679 2023-11-19T01:27:46,408 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(476): Truncating exising target table 'test-1700357184679', preserving region splits 2023-11-19T01:27:46,409 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:27:46,411 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:53400, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:27:46,411 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable test-1700357184679 2023-11-19T01:27:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=83, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=test-1700357184679 2023-11-19T01:27:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=83 2023-11-19T01:27:46,415 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1700357184679","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357266415"}]},"ts":"1700357266415"} 2023-11-19T01:27:46,416 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1700357184679, state=DISABLING in hbase:meta 2023-11-19T01:27:46,418 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set test-1700357184679 to state=DISABLING 2023-11-19T01:27:46,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1700357184679, region=98c2f2b8120be772f779a6f83df59517, UNASSIGN}] 2023-11-19T01:27:46,420 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=test-1700357184679, region=98c2f2b8120be772f779a6f83df59517, UNASSIGN 2023-11-19T01:27:46,420 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=84 updating hbase:meta row=98c2f2b8120be772f779a6f83df59517, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:46,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE; CloseRegionProcedure 98c2f2b8120be772f779a6f83df59517, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:27:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=83 2023-11-19T01:27:46,572 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:46,573 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(119): Close 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:46,573 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1668): Closing 98c2f2b8120be772f779a6f83df59517, disabling compactions & flushes 2023-11-19T01:27:46,573 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1690): Closing region test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. 2023-11-19T01:27:46,573 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. 2023-11-19T01:27:46,573 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1778): Acquired close lock on test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. after waiting 0 ms 2023-11-19T01:27:46,573 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1788): Updates disabled for region test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. 2023-11-19T01:27:46,579 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/recovered.edits/10.seqid, newMaxSeqId=10, maxSeqId=1 2023-11-19T01:27:46,580 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:27:46,581 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1909): Closed test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517. 2023-11-19T01:27:46,581 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1622): Region close journal for 98c2f2b8120be772f779a6f83df59517: 2023-11-19T01:27:46,582 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(163): Closed 98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:46,582 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=84 updating hbase:meta row=98c2f2b8120be772f779a6f83df59517, regionState=CLOSED 2023-11-19T01:27:46,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=85, resume processing ppid=84 2023-11-19T01:27:46,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=85, ppid=84, state=SUCCESS; CloseRegionProcedure 98c2f2b8120be772f779a6f83df59517, server=jenkins-hbase6.apache.org,44997,1700357176784 in 163 msec 2023-11-19T01:27:46,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=84, resume processing ppid=83 2023-11-19T01:27:46,586 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=84, ppid=83, state=SUCCESS; TransitRegionStateProcedure table=test-1700357184679, region=98c2f2b8120be772f779a6f83df59517, UNASSIGN in 167 msec 2023-11-19T01:27:46,586 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1700357184679","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357266586"}]},"ts":"1700357266586"} 2023-11-19T01:27:46,587 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1700357184679, state=DISABLED in hbase:meta 2023-11-19T01:27:46,593 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set test-1700357184679 to state=DISABLED 2023-11-19T01:27:46,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=83, state=SUCCESS; DisableTableProcedure table=test-1700357184679 in 183 msec 2023-11-19T01:27:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=83 2023-11-19T01:27:46,718 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$6(2481): Client=jenkins//172.31.2.20 truncate test-1700357184679 2023-11-19T01:27:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=86, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=test-1700357184679 preserveSplits=true) 2023-11-19T01:27:46,720 DEBUG [PEWorker-4 {}] procedure.TruncateTableProcedure(87): waiting for 'test-1700357184679' regions in transition 2023-11-19T01:27:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=86 2023-11-19T01:27:46,723 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:46,725 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f, FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/recovered.edits] 2023-11-19T01:27:46,728 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/archive/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ 2023-11-19T01:27:46,731 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/recovered.edits/10.seqid to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/archive/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/recovered.edits/10.seqid 2023-11-19T01:27:46,732 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:46,732 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(293): Archived test-1700357184679 regions 2023-11-19T01:27:46,737 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of test-1700357184679 from hbase:meta 2023-11-19T01:27:46,738 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(387): Removing 'test-1700357184679' descriptor. 2023-11-19T01:27:46,739 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(377): Removing 'test-1700357184679' from region states. 2023-11-19T01:27:46,739 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357266739"}]},"ts":"9223372036854775807"} 2023-11-19T01:27:46,741 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:27:46,741 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 98c2f2b8120be772f779a6f83df59517, NAME => 'test-1700357184679,,1700357223310.98c2f2b8120be772f779a6f83df59517.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:27:46,741 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(381): Marking 'test-1700357184679' as deleted. 2023-11-19T01:27:46,741 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"test-1700357184679","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1700357266741"}]},"ts":"9223372036854775807"} 2023-11-19T01:27:46,742 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1678): Deleted table test-1700357184679 state from META 2023-11-19T01:27:46,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=86 2023-11-19T01:27:47,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=86 2023-11-19T01:27:47,151 INFO [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => b696d85cf645e95a934c0b80f486a6f6, NAME => 'test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test-1700357184679', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:27:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=86 2023-11-19T01:27:47,557 DEBUG [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(888): Instantiated test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:27:47,557 DEBUG [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1668): Closing b696d85cf645e95a934c0b80f486a6f6, disabling compactions & flushes 2023-11-19T01:27:47,558 INFO [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1690): Closing region test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6. 2023-11-19T01:27:47,558 DEBUG [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6. 2023-11-19T01:27:47,558 DEBUG [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6. after waiting 0 ms 2023-11-19T01:27:47,558 DEBUG [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6. 2023-11-19T01:27:47,558 INFO [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1909): Closed test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6. 2023-11-19T01:27:47,558 DEBUG [RegionOpenAndInit-test-1700357184679-pool-0 {}] regionserver.HRegion(1622): Region close journal for b696d85cf645e95a934c0b80f486a6f6: 2023-11-19T01:27:47,559 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1700357267559"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357267559"}]},"ts":"1700357267559"} 2023-11-19T01:27:47,560 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:27:47,561 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1700357184679","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357267561"}]},"ts":"1700357267561"} 2023-11-19T01:27:47,562 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1700357184679, state=ENABLING in hbase:meta 2023-11-19T01:27:47,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1700357184679, region=b696d85cf645e95a934c0b80f486a6f6, ASSIGN}] 2023-11-19T01:27:47,568 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=87, ppid=86, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=test-1700357184679, region=b696d85cf645e95a934c0b80f486a6f6, ASSIGN 2023-11-19T01:27:47,569 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=87, ppid=86, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=test-1700357184679, region=b696d85cf645e95a934c0b80f486a6f6, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,44997,1700357176784; forceNewPlan=false, retain=false 2023-11-19T01:27:47,720 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=87 updating hbase:meta row=b696d85cf645e95a934c0b80f486a6f6, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:47,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE; OpenRegionProcedure b696d85cf645e95a934c0b80f486a6f6, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:27:47,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=86 2023-11-19T01:27:47,874 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:47,878 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(130): Open test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6. 2023-11-19T01:27:47,878 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7254): Opening region: {ENCODED => b696d85cf645e95a934c0b80f486a6f6, NAME => 'test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:27:47,878 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:27:47,878 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-1700357184679 b696d85cf645e95a934c0b80f486a6f6 2023-11-19T01:27:47,878 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(888): Instantiated test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:27:47,878 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7296): checking encryption for b696d85cf645e95a934c0b80f486a6f6 2023-11-19T01:27:47,879 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7299): checking classloading for b696d85cf645e95a934c0b80f486a6f6 2023-11-19T01:27:47,880 INFO [StoreOpener-b696d85cf645e95a934c0b80f486a6f6-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region b696d85cf645e95a934c0b80f486a6f6 2023-11-19T01:27:47,881 INFO [StoreOpener-b696d85cf645e95a934c0b80f486a6f6-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b696d85cf645e95a934c0b80f486a6f6 columnFamilyName f 2023-11-19T01:27:47,881 DEBUG [StoreOpener-b696d85cf645e95a934c0b80f486a6f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:27:47,882 INFO [StoreOpener-b696d85cf645e95a934c0b80f486a6f6-1 {}] regionserver.HStore(324): Store=b696d85cf645e95a934c0b80f486a6f6/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:27:47,882 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6 2023-11-19T01:27:47,883 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6 2023-11-19T01:27:47,884 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1076): writing seq id for b696d85cf645e95a934c0b80f486a6f6 2023-11-19T01:27:47,886 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:27:47,887 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): Opened b696d85cf645e95a934c0b80f486a6f6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10483174080, jitterRate=-0.023678332567214966}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:27:47,887 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(992): Region open journal for b696d85cf645e95a934c0b80f486a6f6: 2023-11-19T01:27:47,888 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2556): Post open deploy tasks for test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6., pid=88, masterSystemTime=1700357267874 2023-11-19T01:27:47,889 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2583): Finished post open deploy task for test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6. 2023-11-19T01:27:47,889 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(158): Opened test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6. 2023-11-19T01:27:47,890 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=87 updating hbase:meta row=b696d85cf645e95a934c0b80f486a6f6, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:47,892 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=88, resume processing ppid=87 2023-11-19T01:27:47,892 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=88, ppid=87, state=SUCCESS; OpenRegionProcedure b696d85cf645e95a934c0b80f486a6f6, server=jenkins-hbase6.apache.org,44997,1700357176784 in 169 msec 2023-11-19T01:27:47,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=87, resume processing ppid=86 2023-11-19T01:27:47,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=87, ppid=86, state=SUCCESS; TransitRegionStateProcedure table=test-1700357184679, region=b696d85cf645e95a934c0b80f486a6f6, ASSIGN in 325 msec 2023-11-19T01:27:47,894 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"test-1700357184679","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357267894"}]},"ts":"1700357267894"} 2023-11-19T01:27:47,895 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=test-1700357184679, state=ENABLED in hbase:meta 2023-11-19T01:27:47,897 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(145): truncate 'test-1700357184679' completed 2023-11-19T01:27:47,898 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=86, state=SUCCESS; TruncateTableProcedure (table=test-1700357184679 preserveSplits=true) in 1.1790 sec 2023-11-19T01:27:48,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=86 2023-11-19T01:27:48,831 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:42739/backupUT/backup_1700357256873/default/test-1700357184679/archive/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517 from hbase tables test-1700357184679 to tables test-1700357184679 2023-11-19T01:27:48,831 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1700357184679 into test-1700357184679 2023-11-19T01:27:48,833 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-test-1700357184679-1700357268831 from hdfs://localhost:42739/backupUT/backup_1700357256873/default/test-1700357184679/archive/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517 2023-11-19T01:27:48,839 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a9913d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:48,840 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:27:48,841 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:53406, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:27:48,843 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-19T01:27:48,843 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table test-1700357184679 2023-11-19T01:27:48,846 DEBUG [hconnection-0x605eca4e-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:48,848 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:40958, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:48,849 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for test-1700357184679: 2023-11-19T01:27:48,849 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-11-19T01:27:48,849 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/partitions_c9511451-d683-4b3b-8fed-1ecdfe2d6b6f 2023-11-19T01:27:49,258 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:49,259 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:49,259 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:49,259 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:50,309 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-6138939638974513357.jar 2023-11-19T01:27:50,310 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:50,310 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:50,311 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:50,311 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:50,312 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:50,312 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:50,313 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:50,313 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:50,314 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:50,315 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:27:50,316 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:27:50,317 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:27:50,317 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:27:50,318 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:27:50,319 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:27:50,320 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:27:50,320 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:27:50,321 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:27:50,321 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:27:50,322 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:27:50,322 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:27:50,323 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:50,324 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:50,324 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:50,324 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:50,325 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:50,325 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:27:50,326 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:27:50,326 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: test-1700357184679 2023-11-19T01:27:50,326 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:50,326 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-11-19T01:27:50,326 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-11-19T01:27:50,327 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:27:50,343 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-11-19T01:27:50,345 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:27:50,436 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357270397/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:27:50,437 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357270397/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:27:50,487 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:42739/backupUT/backup_1700357256873/default/test-1700357184679/archive/data/default/test-1700357184679/98c2f2b8120be772f779a6f83df59517/f/f67a9a1682c048d4a4b4d737b4cec0ce_SeqId_4_ 2023-11-19T01:27:50,491 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-11-19T01:27:50,521 DEBUG [pool-332-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c98b741, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:50,523 DEBUG [hconnection-0x546e23d5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:50,525 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44504, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:50,529 DEBUG [pool-332-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:50,529 DEBUG [pool-332-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-11-19T01:27:50,530 DEBUG [pool-332-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase6.apache.org 2023-11-19T01:27:50,537 DEBUG [pool-332-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-test-1700357184679-1700357268831/_temporary/0/_temporary/attempt_local1006382434_0011_r_000000_0/f/d619ec71af0745cb901d76cb10420878 is 37, key is row10/f:q1/1700357187320/Put/seqid=0 2023-11-19T01:27:50,539 WARN [Thread-1397 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase6.apache.org:44997] Specified favored nodes: [jenkins-hbase6.apache.org:44997] 2023-11-19T01:27:51,482 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-test-1700357184679-1700357268831 2023-11-19T01:27:51,488 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25ac6d75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:51,489 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:51,490 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44516, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:51,496 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-test-1700357184679-1700357268831/_SUCCESS 2023-11-19T01:27:51,499 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:27:51,500 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43900, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:27:51,509 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-test-1700357184679-1700357268831/f/d619ec71af0745cb901d76cb10420878 first=Optional[row0] last=Optional[row98] 2023-11-19T01:27:51,510 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6., hostname=jenkins-hbase6.apache.org,44997,1700357176784, seqNum=2 for row with hfile group [{f,hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-test-1700357184679-1700357268831/f/d619ec71af0745cb901d76cb10420878}] 2023-11-19T01:27:51,517 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-test-1700357184679-1700357268831/f/d619ec71af0745cb901d76cb10420878 for inclusion in b696d85cf645e95a934c0b80f486a6f6/f 2023-11-19T01:27:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-11-19T01:27:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(625): Region bounds: first= last= 2023-11-19T01:27:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegion(2520): Flush status journal for b696d85cf645e95a934c0b80f486a6f6: 2023-11-19T01:27:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-test-1700357184679-1700357268831/f/d619ec71af0745cb901d76cb10420878 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__test-1700357184679__n7vlhpuibcshj434mcfaicd0m83ohlq9b1adqvmb9hh6g2p54pgkkopi46suu4ej/f/d619ec71af0745cb901d76cb10420878 2023-11-19T01:27:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__test-1700357184679__n7vlhpuibcshj434mcfaicd0m83ohlq9b1adqvmb9hh6g2p54pgkkopi46suu4ej/f/d619ec71af0745cb901d76cb10420878 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6/f/8cf00b857f704ccba204d7492145d044_SeqId_4_ 2023-11-19T01:27:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29009556, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:27:51,532 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43916, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:27:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:51,536 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44526, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(428): write bulk load descriptor to backup test-1700357184679 with 1 entries 2023-11-19T01:27:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(1602): writing raw bulk path hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6/f/8cf00b857f704ccba204d7492145d044_SeqId_4_ for test-1700357184679 b696d85cf645e95a934c0b80f486a6f6 2023-11-19T01:27:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(435): written 1 rows for bulk load of test-1700357184679 2023-11-19T01:27:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:51,555 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__test-1700357184679__n7vlhpuibcshj434mcfaicd0m83ohlq9b1adqvmb9hh6g2p54pgkkopi46suu4ej/f/d619ec71af0745cb901d76cb10420878 into b696d85cf645e95a934c0b80f486a6f6/f as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6/f/8cf00b857f704ccba204d7492145d044_SeqId_4_ - updating store file list. 2023-11-19T01:27:51,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6/f/8cf00b857f704ccba204d7492145d044_SeqId_4_ into b696d85cf645e95a934c0b80f486a6f6/f 2023-11-19T01:27:51,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__test-1700357184679__n7vlhpuibcshj434mcfaicd0m83ohlq9b1adqvmb9hh6g2p54pgkkopi46suu4ej/f/d619ec71af0745cb901d76cb10420878 into b696d85cf645e95a934c0b80f486a6f6/f (new location: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6/f/8cf00b857f704ccba204d7492145d044_SeqId_4_) 2023-11-19T01:27:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__test-1700357184679__n7vlhpuibcshj434mcfaicd0m83ohlq9b1adqvmb9hh6g2p54pgkkopi46suu4ej/f/d619ec71af0745cb901d76cb10420878 2023-11-19T01:27:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73d9fb69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:27:51,571 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43932, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-19T01:27:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:51,576 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44532, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-11-19T01:27:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup test-1700357184679 with 1 entries 2023-11-19T01:27:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(1523): writing done bulk path hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6/f/8cf00b857f704ccba204d7492145d044_SeqId_4_ for test-1700357184679 b696d85cf645e95a934c0b80f486a6f6 2023-11-19T01:27:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(414): written 1 rows for bulk load of test-1700357184679 2023-11-19T01:27:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:51,599 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:51,599 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-11-19T01:27:51,599 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-19T01:27:51,599 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(233): Backup: backup_1700357256873 hdfs://localhost:42739/backupUT/backup_1700357256873/default/test-1700357184679/ 2023-11-19T01:27:51,599 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-11-19T01:27:51,600 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:51,622 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleOverwriteCommand Thread=605 (was 596) - Thread LEAK? -, OpenFileDescriptor=720 (was 710) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=115 (was 149), ProcessCount=167 (was 166) - ProcessCount LEAK? -, AvailableMemoryMB=3596 (was 3666) 2023-11-19T01:27:51,622 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-11-19T01:27:51,638 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleDNECommand Thread=605, OpenFileDescriptor=720, MaxFileDescriptor=60000, SystemLoadAverage=115, ProcessCount=167, AvailableMemoryMB=3595 2023-11-19T01:27:51,638 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-11-19T01:27:51,638 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestFullRestore(322): test restore fails on multiple tables that do not exist: command-line 2023-11-19T01:27:51,645 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dc73f20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:51,649 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:51,650 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44542, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:51,652 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:27:51,654 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43936, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:27:51,670 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:27:51,676 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:27:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357271676 (current time:1700357271676). 2023-11-19T01:27:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:27:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:27:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78d20b73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:51,690 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44556, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:27:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=89, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:27:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 89 2023-11-19T01:27:51,695 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:27:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=89 2023-11-19T01:27:51,696 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:27:51,699 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:27:51,702 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:27:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=89 2023-11-19T01:27:51,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=89 2023-11-19T01:27:52,108 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:27:52,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc}] 2023-11-19T01:27:52,110 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=90, ppid=89, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:27:52,260 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:52,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=90 2023-11-19T01:27:52,261 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:27:52,261 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegion(2819): Flushing 322466352cac3ade468ab2ce8e706acc 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-11-19T01:27:52,265 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/298b24d86eff42c285c52df630edc72c is 169, key is trslm:hdfs://localhost:42739/backupUT\x00test-1700357184679/meta:log-roll-map/1700357265944/Put/seqid=0 2023-11-19T01:27:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=89 2023-11-19T01:27:52,669 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/298b24d86eff42c285c52df630edc72c 2023-11-19T01:27:52,677 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/1b2851533774422da3cbc0adf48207b8 is 452, key is session:backup_1700357256873/session:context/1700357266366/Put/seqid=0 2023-11-19T01:27:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=89 2023-11-19T01:27:53,081 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=72 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/1b2851533774422da3cbc0adf48207b8 2023-11-19T01:27:53,087 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/298b24d86eff42c285c52df630edc72c as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/298b24d86eff42c285c52df630edc72c 2023-11-19T01:27:53,092 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/298b24d86eff42c285c52df630edc72c, entries=4, sequenceid=72, filesize=5.5 K 2023-11-19T01:27:53,093 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/1b2851533774422da3cbc0adf48207b8 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/1b2851533774422da3cbc0adf48207b8 2023-11-19T01:27:53,097 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/1b2851533774422da3cbc0adf48207b8, entries=2, sequenceid=72, filesize=5.4 K 2023-11-19T01:27:53,098 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 322466352cac3ade468ab2ce8e706acc in 837ms, sequenceid=72, compaction requested=true 2023-11-19T01:27:53,098 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.HRegion(2520): Flush status journal for 322466352cac3ade468ab2ce8e706acc: 2023-11-19T01:27:53,098 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. for snapshot_backup_system completed. 2023-11-19T01:27:53,099 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:27:53,099 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:27:53,099 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/83b8d86696ce4c728d410b325dfe4099, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/817bbb8696494171be63b200e480be74, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/298b24d86eff42c285c52df630edc72c] hfiles 2023-11-19T01:27:53,099 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (1/6): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757 for snapshot=snapshot_backup_system 2023-11-19T01:27:53,099 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (2/6): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63 for snapshot=snapshot_backup_system 2023-11-19T01:27:53,099 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (3/6): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2 for snapshot=snapshot_backup_system 2023-11-19T01:27:53,099 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (4/6): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/83b8d86696ce4c728d410b325dfe4099 for snapshot=snapshot_backup_system 2023-11-19T01:27:53,099 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (5/6): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/817bbb8696494171be63b200e480be74 for snapshot=snapshot_backup_system 2023-11-19T01:27:53,099 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (6/6): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/298b24d86eff42c285c52df630edc72c for snapshot=snapshot_backup_system 2023-11-19T01:27:53,099 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/05f6b05b598842a0a5af3e603f2dfa9f, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/6e39103009374e778431dfd7316f5a4c, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/1b2851533774422da3cbc0adf48207b8] hfiles 2023-11-19T01:27:53,099 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99 for snapshot=snapshot_backup_system 2023-11-19T01:27:53,099 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff for snapshot=snapshot_backup_system 2023-11-19T01:27:53,099 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460 for snapshot=snapshot_backup_system 2023-11-19T01:27:53,099 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1 for snapshot=snapshot_backup_system 2023-11-19T01:27:53,099 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/05f6b05b598842a0a5af3e603f2dfa9f for snapshot=snapshot_backup_system 2023-11-19T01:27:53,099 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/6e39103009374e778431dfd7316f5a4c for snapshot=snapshot_backup_system 2023-11-19T01:27:53,099 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/1b2851533774422da3cbc0adf48207b8 for snapshot=snapshot_backup_system 2023-11-19T01:27:53,504 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:27:53,504 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=90 2023-11-19T01:27:53,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=90 2023-11-19T01:27:53,505 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:27:53,506 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=90, ppid=89, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:27:53,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=90, resume processing ppid=89 2023-11-19T01:27:53,508 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:27:53,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=90, ppid=89, state=SUCCESS; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc in 1.3990 sec 2023-11-19T01:27:53,509 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:27:53,509 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:27:53,509 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:27:53,510 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:27:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=89 2023-11-19T01:27:53,922 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:27:53,932 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:27:53,932 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:27:53,933 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=89, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:27:53,934 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 89 2023-11-19T01:27:53,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=89, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=89, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2420 sec 2023-11-19T01:27:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=89 2023-11-19T01:27:55,802 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(115): Backup backup_1700357271645 started at 1700357275802. 2023-11-19T01:27:55,803 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(119): Backup session backup_1700357271645 has been started. 2023-11-19T01:27:55,808 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-19T01:27:55,809 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:27:55,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:27:55,809 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:27:55,809 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:27:55,810 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:27:55,810 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:27:55,810 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:27:55,812 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:55,812 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:27:55,812 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:27:55,812 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:27:55,812 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:27:55,813 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:27:55,813 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:27:55,813 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:27:55,813 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:27:55,813 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:27:55,813 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:27:55,813 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:27:55,814 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:27:55,814 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:27:55,814 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:27:55,816 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:55,816 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:55,816 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:27:55,817 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:27:55,817 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:27:55,817 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:27:55,817 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:27:55,818 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:55,818 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:55,818 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:27:55,819 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:27:55,819 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@213d9678[Count = 0] remaining members to acquire global barrier 2023-11-19T01:27:55,819 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:27:55,819 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:55,821 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:55,821 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:27:55,821 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:27:55,821 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:27:55,821 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:27:55,821 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:55,821 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357261034 highest: 1700357261034 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:55,821 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784:(num 1700357261034) roll requested 2023-11-19T01:27:55,835 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357275821, exclude list is [], retry=0 2023-11-19T01:27:55,839 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:27:55,841 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357261034 with entries=15, filesize=4.76 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357275821 2023-11-19T01:27:55,841 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:27:55,841 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357261034 is not closed yet, will try archiving it next time 2023-11-19T01:27:55,841 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta:.meta(num 1700357261052) roll requested 2023-11-19T01:27:55,841 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357247588 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/oldWALs/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357247588 2023-11-19T01:27:55,853 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357275841.meta, exclude list is [], retry=0 2023-11-19T01:27:55,856 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:27:55,857 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357261052.meta with entries=12, filesize=3.07 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357275841.meta 2023-11-19T01:27:55,857 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:27:55,857 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357261052.meta is not closed yet, will try archiving it next time 2023-11-19T01:27:55,862 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-11-19T01:27:55,862 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357275821 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:55,870 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:27:55,870 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,44997,1700357176784' in zk 2023-11-19T01:27:55,872 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:27:55,872 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:55,872 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:27:55,872 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:27:55,872 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:27:55,872 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:27:55,873 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:27:55,873 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:27:55,874 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:55,874 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:55,874 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:27:55,875 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:55,875 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:55,876 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,44997,1700357176784': 2023-11-19T01:27:55,876 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:27:55,876 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:27:55,876 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:27:55,876 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:27:55,876 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:27:55,876 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:27:55,878 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:27:55,878 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:27:55,878 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:27:55,878 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:27:55,878 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:27:55,878 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:27:55,878 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:27:55,878 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:27:55,878 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:55,879 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:27:55,879 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:27:55,879 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:55,879 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:55,880 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:27:55,880 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:27:55,881 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:55,884 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:27:55,884 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:27:55,884 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:27:55,884 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:27:55,884 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:27:55,885 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:27:55,885 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:27:55,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-19T01:27:55,988 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357275988_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } 2023-11-19T01:27:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357275988 (current time:1700357275988). 2023-11-19T01:27:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:27:55,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357275988_ns2_test-17003571846791 VERSION not specified, setting to 2 2023-11-19T01:27:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3acfcb5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:55,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:55,997 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44572, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:27:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=91, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1700357275988_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } 2023-11-19T01:27:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357275988_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 }, snapshot procedure id = 91 2023-11-19T01:27:56,001 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1700357275988_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:27:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=91 2023-11-19T01:27:56,001 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1700357275988_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:27:56,004 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357275988_ns2_test-17003571846791 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:27:56,006 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1700357275988_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:27:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=91 2023-11-19T01:27:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=91 2023-11-19T01:27:56,411 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1700357275988_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:27:56,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; SnapshotRegionProcedure 4ef412d65669b2fe1684b7fdf6ee1f7d}] 2023-11-19T01:27:56,413 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=92, ppid=91, state=RUNNABLE; SnapshotRegionProcedure 4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:27:56,564 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:56,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=92 2023-11-19T01:27:56,564 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. 2023-11-19T01:27:56,565 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.HRegion(2520): Flush status journal for 4ef412d65669b2fe1684b7fdf6ee1f7d: 2023-11-19T01:27:56,565 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. for snapshot_1700357275988_ns2_test-17003571846791 completed. 2023-11-19T01:27:56,565 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(240): Storing 'ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d.' region-info for snapshot=snapshot_1700357275988_ns2_test-17003571846791 2023-11-19T01:27:56,565 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:27:56,565 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_] hfiles 2023-11-19T01:27:56,565 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_ for snapshot=snapshot_1700357275988_ns2_test-17003571846791 2023-11-19T01:27:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=91 2023-11-19T01:27:56,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. 2023-11-19T01:27:56,970 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=92 2023-11-19T01:27:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=92 2023-11-19T01:27:56,971 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357275988_ns2_test-17003571846791 on region 4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:27:56,972 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=92, ppid=91, state=RUNNABLE; SnapshotRegionProcedure 4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:27:56,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=92, resume processing ppid=91 2023-11-19T01:27:56,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=92, ppid=91, state=SUCCESS; SnapshotRegionProcedure 4ef412d65669b2fe1684b7fdf6ee1f7d in 562 msec 2023-11-19T01:27:56,974 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1700357275988_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:27:56,975 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1700357275988_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:27:56,975 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1700357275988_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:27:56,975 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357275988_ns2_test-17003571846791 2023-11-19T01:27:56,976 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357275988_ns2_test-17003571846791 2023-11-19T01:27:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=91 2023-11-19T01:27:57,385 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1700357275988_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:27:57,390 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1700357275988_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:27:57,391 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357275988_ns2_test-17003571846791 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357275988_ns2_test-17003571846791 2023-11-19T01:27:57,392 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=91, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1700357275988_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:27:57,392 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357275988_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 }, snapshot procedure id = 91 2023-11-19T01:27:57,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=91, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=91, snapshot={ ss=snapshot_1700357275988_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } in 1.3940 sec 2023-11-19T01:27:58,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=91 2023-11-19T01:27:58,107 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357278106_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } 2023-11-19T01:27:58,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357278107 (current time:1700357278107). 2023-11-19T01:27:58,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:27:58,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357278106_ns3_test-17003571846792 VERSION not specified, setting to 2 2023-11-19T01:27:58,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ccf9656, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:27:58,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:27:58,115 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44584, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:27:58,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:27:58,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:27:58,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=93, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1700357278106_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } 2023-11-19T01:27:58,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357278106_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 }, snapshot procedure id = 93 2023-11-19T01:27:58,119 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1700357278106_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:27:58,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=93 2023-11-19T01:27:58,120 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1700357278106_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:27:58,122 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357278106_ns3_test-17003571846792 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_362] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_362] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:27:58,124 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1700357278106_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:27:58,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=93 2023-11-19T01:27:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=93 2023-11-19T01:27:58,530 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1700357278106_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:27:58,530 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE; SnapshotRegionProcedure b78f59bec877d4d278f3912d6fb4631e}] 2023-11-19T01:27:58,532 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=94, ppid=93, state=RUNNABLE; SnapshotRegionProcedure b78f59bec877d4d278f3912d6fb4631e 2023-11-19T01:27:58,682 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:27:58,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=94 2023-11-19T01:27:58,683 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. 2023-11-19T01:27:58,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.HRegion(2520): Flush status journal for b78f59bec877d4d278f3912d6fb4631e: 2023-11-19T01:27:58,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. for snapshot_1700357278106_ns3_test-17003571846792 completed. 2023-11-19T01:27:58,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] snapshot.SnapshotManifest(240): Storing 'ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e.' region-info for snapshot=snapshot_1700357278106_ns3_test-17003571846792 2023-11-19T01:27:58,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:27:58,684 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-19T01:27:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=93 2023-11-19T01:27:59,090 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. 2023-11-19T01:27:59,090 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=94 2023-11-19T01:27:59,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=94 2023-11-19T01:27:59,091 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357278106_ns3_test-17003571846792 on region b78f59bec877d4d278f3912d6fb4631e 2023-11-19T01:27:59,092 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=94, ppid=93, state=RUNNABLE; SnapshotRegionProcedure b78f59bec877d4d278f3912d6fb4631e 2023-11-19T01:27:59,094 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=94, resume processing ppid=93 2023-11-19T01:27:59,094 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=94, ppid=93, state=SUCCESS; SnapshotRegionProcedure b78f59bec877d4d278f3912d6fb4631e in 563 msec 2023-11-19T01:27:59,094 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1700357278106_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:27:59,094 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1700357278106_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:27:59,095 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1700357278106_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:27:59,095 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357278106_ns3_test-17003571846792 2023-11-19T01:27:59,095 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357278106_ns3_test-17003571846792 2023-11-19T01:27:59,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=93 2023-11-19T01:27:59,504 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1700357278106_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:27:59,508 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1700357278106_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:27:59,509 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357278106_ns3_test-17003571846792 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357278106_ns3_test-17003571846792 2023-11-19T01:27:59,510 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=93, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1700357278106_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:27:59,511 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357278106_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 }, snapshot procedure id = 93 2023-11-19T01:27:59,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=93, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=93, snapshot={ ss=snapshot_1700357278106_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } in 1.3940 sec 2023-11-19T01:28:00,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=93 2023-11-19T01:28:00,224 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1700357271645 2023-11-19T01:28:00,224 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-19T01:28:00,224 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-11-19T01:28:00,224 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357271645_ns2:test-17003571846791 2023-11-19T01:28:00,224 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357275988_ns2_test-17003571846791 to hdfs://localhost:42739/backupUT/backup_1700357271645/ns2/test-17003571846791/ 2023-11-19T01:28:00,224 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:28:00,248 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:28:00,248 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,44889,1700357173572_847815212_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42739/backupUT/backup_1700357271645/ns2/test-17003571846791, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42739/backupUT/backup_1700357271645/ns2/test-17003571846791/.hbase-snapshot/.tmp/snapshot_1700357275988_ns2_test-17003571846791 2023-11-19T01:28:00,248 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af, snapshotDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357275988_ns2_test-17003571846791. 2023-11-19T01:28:00,258 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357275988_ns2_test-17003571846791 to hdfs://localhost:42739/backupUT/backup_1700357271645/ns2/test-17003571846791/.hbase-snapshot/.tmp/snapshot_1700357275988_ns2_test-17003571846791 2023-11-19T01:28:00,679 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:00,680 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:00,680 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:00,681 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:01,670 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-6946420089726497547.jar 2023-11-19T01:28:01,671 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:01,671 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:01,672 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:01,672 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:01,672 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:01,673 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:01,673 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:01,673 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:01,674 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:01,674 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:28:01,675 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:28:01,675 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:28:01,676 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:28:01,676 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:28:01,677 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:28:01,677 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:28:01,677 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:28:01,678 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:28:01,678 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:28:01,679 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:28:01,679 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:28:01,680 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:01,680 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:01,681 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:01,681 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:01,681 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:01,682 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:01,682 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:01,725 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357275988_ns2_test-17003571846791' hfile list 2023-11-19T01:28:01,728 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-19T01:28:01,799 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357281764/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:28:01,799 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357281764/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:28:01,857 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-19T01:28:01,904 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_ output=hdfs://localhost:42739/backupUT/backup_1700357271645/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_ 2023-11-19T01:28:01,904 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 4.160M/sec 2023-11-19T01:28:02,832 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:28:02,833 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:28:02,838 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357275988_ns2_test-17003571846791 2023-11-19T01:28:02,839 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357275988_ns2_test-17003571846791 finished. 2023-11-19T01:28:02,839 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357271645_ns3:test-17003571846792 2023-11-19T01:28:02,839 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357278106_ns3_test-17003571846792 to hdfs://localhost:42739/backupUT/backup_1700357271645/ns3/test-17003571846792/ 2023-11-19T01:28:02,839 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:28:02,862 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:28:02,862 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,44889,1700357173572_-1994841860_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42739/backupUT/backup_1700357271645/ns3/test-17003571846792, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42739/backupUT/backup_1700357271645/ns3/test-17003571846792/.hbase-snapshot/.tmp/snapshot_1700357278106_ns3_test-17003571846792 2023-11-19T01:28:02,862 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af, snapshotDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357278106_ns3_test-17003571846792. 2023-11-19T01:28:02,867 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357278106_ns3_test-17003571846792 to hdfs://localhost:42739/backupUT/backup_1700357271645/ns3/test-17003571846792/.hbase-snapshot/.tmp/snapshot_1700357278106_ns3_test-17003571846792 2023-11-19T01:28:03,280 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:03,281 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:03,281 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:03,282 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:04,267 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-3392949699823633451.jar 2023-11-19T01:28:04,268 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:04,268 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:04,269 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:04,269 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:04,269 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:04,270 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:04,270 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:04,270 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:04,271 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:04,271 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:28:04,272 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:28:04,272 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:28:04,273 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:28:04,273 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:28:04,274 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:28:04,274 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:28:04,274 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:28:04,275 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:28:04,275 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:28:04,276 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:28:04,276 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:28:04,276 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:04,277 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:04,278 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:04,278 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:04,278 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:04,279 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:04,280 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:04,327 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357278106_ns3_test-17003571846792' hfile list 2023-11-19T01:28:04,400 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357284366/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:28:04,400 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357284366/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:28:05,427 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:28:05,428 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:28:05,433 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357278106_ns3_test-17003571846792 2023-11-19T01:28:05,433 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357278106_ns3_test-17003571846792 finished. 2023-11-19T01:28:05,433 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1034): ns2:test-17003571846791 2023-11-19T01:28:05,433 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1034): ns3:test-17003571846792 2023-11-19T01:28:05,439 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357271645 2023-11-19T01:28:05,439 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:28:05,844 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42739/backupUT/backup_1700357271645/.backup.manifest 2023-11-19T01:28:05,844 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357271645 2023-11-19T01:28:05,844 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:28:06,250 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42739/backupUT/backup_1700357271645/.backup.manifest 2023-11-19T01:28:06,250 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(358): Backup backup_1700357271645 finished: type=FULL,tablelist=ns2:test-17003571846791;ns3:test-17003571846792,targetRootDir=hdfs://localhost:42739/backupUT,startts=1700357275802,completets=1700357285439,bytescopied=0 2023-11-19T01:28:06,251 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-19T01:28:06,251 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357275988_ns2_test-17003571846791 2023-11-19T01:28:06,253 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357275988_ns2_test-17003571846791" 2023-11-19T01:28:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357275988_ns2_test-17003571846791 2023-11-19T01:28:06,256 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357275988_ns2_test-17003571846791 for backup backup_1700357271645 succeeded. 2023-11-19T01:28:06,256 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357278106_ns3_test-17003571846792 2023-11-19T01:28:06,257 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357278106_ns3_test-17003571846792" 2023-11-19T01:28:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357278106_ns3_test-17003571846792 2023-11-19T01:28:06,259 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357278106_ns3_test-17003571846792 for backup backup_1700357271645 succeeded. 2023-11-19T01:28:06,260 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:28:06,264 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:28:06,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:28:06,267 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:28:06,268 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:28:06,273 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(376): Backup backup_1700357271645 completed. 2023-11-19T01:28:06,273 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client You have to specify either set name or table list to restore Usage: hbase restore [options] backup_path Path to a backup destination root backup_id Backup image ID to restore table(s) Comma-separated list of tables to restore Options: -c Check restore sequence and dependencies only (does not execute the command) -d Enable debug loggings -h,--help Show usage -m A comma separated list of target tables. If specified, each table in must have a mapping -o Overwrite data if any of the restore target tables exists -q Yarn queue name to run backup restore command on -s Backup set to restore, mutually exclusive with -t (table list) -t Table name list, comma-separated. To enable backup, in hbase-site.xml, set: hbase.backup.enable=true hbase.master.logcleaner.plugins=YOUR_PLUGINS,org.apache.hadoop.hbase.backup.master.BackupLogCleaner hbase.procedure.master.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager hbase.procedure.regionserver.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager hbase.coprocessor.region.classes=YOUR_CLASSES,org.apache.hadoop.hbase.backup.BackupObserver and restart the cluster For more information please see http://hbase.apache.org/book.html#backuprestore 2023-11-19T01:28:06,299 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleDNECommand Thread=598 (was 605), OpenFileDescriptor=711 (was 720), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=98 (was 115), ProcessCount=167 (was 167), AvailableMemoryMB=3602 (was 3595) - AvailableMemoryMB LEAK? - 2023-11-19T01:28:06,299 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=598 is superior to 500 2023-11-19T01:28:06,314 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultiple Thread=598, OpenFileDescriptor=711, MaxFileDescriptor=60000, SystemLoadAverage=98, ProcessCount=167, AvailableMemoryMB=3602 2023-11-19T01:28:06,314 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=598 is superior to 500 2023-11-19T01:28:06,314 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestFullRestore(120): create full backup image on multiple tables 2023-11-19T01:28:06,319 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ae99530, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:06,323 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:06,324 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:47076, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:06,327 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:28:06,328 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:49388, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:28:06,339 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:28:06,344 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:28:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357286344 (current time:1700357286344). 2023-11-19T01:28:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:28:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:28:06,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26084415, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:06,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:06,357 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:47092, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:06,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:06,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:28:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=95, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:28:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 95 2023-11-19T01:28:06,363 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:28:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=95 2023-11-19T01:28:06,364 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:28:06,366 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:28:06,368 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:28:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=95 2023-11-19T01:28:06,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=95 2023-11-19T01:28:06,774 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:28:06,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc}] 2023-11-19T01:28:06,776 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=96, ppid=95, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:28:06,927 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:06,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=96 2023-11-19T01:28:06,928 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:28:06,928 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegion(2819): Flushing 322466352cac3ade468ab2ce8e706acc 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-11-19T01:28:06,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/e32b4b3110b04f12a891660d01b34d8c is 171, key is trslm:hdfs://localhost:42739/backupUT\x00ns2:test-17003571846791/meta:log-roll-map/1700357285436/Put/seqid=0 2023-11-19T01:28:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=95 2023-11-19T01:28:07,336 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/e32b4b3110b04f12a891660d01b34d8c 2023-11-19T01:28:07,345 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/20bd2650ab284e0b9d4e6d4318baa9b6 is 524, key is session:backup_1700357271645/session:context/1700357286267/Put/seqid=0 2023-11-19T01:28:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=95 2023-11-19T01:28:07,749 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/20bd2650ab284e0b9d4e6d4318baa9b6 2023-11-19T01:28:07,755 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/e32b4b3110b04f12a891660d01b34d8c as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/e32b4b3110b04f12a891660d01b34d8c 2023-11-19T01:28:07,760 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/e32b4b3110b04f12a891660d01b34d8c, entries=6, sequenceid=83, filesize=5.8 K 2023-11-19T01:28:07,760 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/20bd2650ab284e0b9d4e6d4318baa9b6 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/20bd2650ab284e0b9d4e6d4318baa9b6 2023-11-19T01:28:07,765 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/20bd2650ab284e0b9d4e6d4318baa9b6, entries=2, sequenceid=83, filesize=5.5 K 2023-11-19T01:28:07,765 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for 322466352cac3ade468ab2ce8e706acc in 837ms, sequenceid=83, compaction requested=true 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.HRegion(2520): Flush status journal for 322466352cac3ade468ab2ce8e706acc: 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. for snapshot_backup_system completed. 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/83b8d86696ce4c728d410b325dfe4099, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/817bbb8696494171be63b200e480be74, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/298b24d86eff42c285c52df630edc72c, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/e32b4b3110b04f12a891660d01b34d8c] hfiles 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (1/7): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757 for snapshot=snapshot_backup_system 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (2/7): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63 for snapshot=snapshot_backup_system 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (3/7): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2 for snapshot=snapshot_backup_system 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (4/7): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/83b8d86696ce4c728d410b325dfe4099 for snapshot=snapshot_backup_system 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (5/7): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/817bbb8696494171be63b200e480be74 for snapshot=snapshot_backup_system 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (6/7): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/298b24d86eff42c285c52df630edc72c for snapshot=snapshot_backup_system 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (7/7): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/e32b4b3110b04f12a891660d01b34d8c for snapshot=snapshot_backup_system 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/05f6b05b598842a0a5af3e603f2dfa9f, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/6e39103009374e778431dfd7316f5a4c, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/1b2851533774422da3cbc0adf48207b8, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/20bd2650ab284e0b9d4e6d4318baa9b6] hfiles 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (1/8): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99 for snapshot=snapshot_backup_system 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (2/8): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff for snapshot=snapshot_backup_system 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (3/8): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460 for snapshot=snapshot_backup_system 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (4/8): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1 for snapshot=snapshot_backup_system 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (5/8): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/05f6b05b598842a0a5af3e603f2dfa9f for snapshot=snapshot_backup_system 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (6/8): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/6e39103009374e778431dfd7316f5a4c for snapshot=snapshot_backup_system 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (7/8): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/1b2851533774422da3cbc0adf48207b8 for snapshot=snapshot_backup_system 2023-11-19T01:28:07,766 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] snapshot.SnapshotManifest(264): Adding reference for file (8/8): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/20bd2650ab284e0b9d4e6d4318baa9b6 for snapshot=snapshot_backup_system 2023-11-19T01:28:08,171 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:28:08,171 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=96 2023-11-19T01:28:08,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=96 2023-11-19T01:28:08,172 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:28:08,175 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=96, ppid=95, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:28:08,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=96, resume processing ppid=95 2023-11-19T01:28:08,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=96, ppid=95, state=SUCCESS; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc in 1.4010 sec 2023-11-19T01:28:08,176 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:28:08,177 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:28:08,177 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:28:08,178 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:28:08,178 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:28:08,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=95 2023-11-19T01:28:08,589 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:28:08,601 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:28:08,601 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:28:08,602 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=95, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:28:08,602 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 95 2023-11-19T01:28:08,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=95, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=95, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2430 sec 2023-11-19T01:28:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=95 2023-11-19T01:28:10,471 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(115): Backup backup_1700357286319 started at 1700357290471. 2023-11-19T01:28:10,472 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(119): Backup session backup_1700357286319 has been started. 2023-11-19T01:28:10,477 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-19T01:28:10,478 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:28:10,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:28:10,478 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:28:10,478 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:28:10,479 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:28:10,479 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:28:10,479 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:28:10,482 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:10,482 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:28:10,482 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:28:10,482 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:28:10,482 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:28:10,483 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:28:10,483 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:28:10,483 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:28:10,483 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:28:10,483 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:28:10,483 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:28:10,483 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:28:10,484 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:28:10,484 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:28:10,484 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:28:10,486 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:28:10,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:10,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:28:10,486 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:28:10,487 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:28:10,487 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:28:10,487 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:28:10,487 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:28:10,488 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:10,488 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:28:10,488 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:28:10,488 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@72a375bd[Count = 0] remaining members to acquire global barrier 2023-11-19T01:28:10,488 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:28:10,488 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:28:10,490 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:28:10,490 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:28:10,490 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:28:10,490 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:28:10,490 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:28:10,490 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:10,490 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357275821 highest: 1700357275821 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:10,490 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784:(num 1700357275821) roll requested 2023-11-19T01:28:10,508 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357290490, exclude list is [], retry=0 2023-11-19T01:28:10,514 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:28:10,515 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357275821 with entries=10, filesize=2.67 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357290490 2023-11-19T01:28:10,515 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:28:10,516 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357275821 is not closed yet, will try archiving it next time 2023-11-19T01:28:10,516 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta:.meta(num 1700357275841) roll requested 2023-11-19T01:28:10,517 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357275821 not finished, retry = 0 2023-11-19T01:28:10,527 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357290516.meta, exclude list is [], retry=0 2023-11-19T01:28:10,530 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:28:10,531 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-11-19T01:28:10,531 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357290490 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:10,532 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357275841.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357290516.meta 2023-11-19T01:28:10,532 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:28:10,532 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357275841.meta is not closed yet, will try archiving it next time 2023-11-19T01:28:10,534 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357275841.meta to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/oldWALs/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357275841.meta 2023-11-19T01:28:10,539 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:28:10,539 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,44997,1700357176784' in zk 2023-11-19T01:28:10,541 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:28:10,542 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:28:10,542 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:28:10,542 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:10,542 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:28:10,542 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:28:10,543 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:28:10,543 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:28:10,543 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:28:10,544 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:10,544 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:28:10,544 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:28:10,545 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:10,545 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,44997,1700357176784': 2023-11-19T01:28:10,545 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:28:10,545 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:28:10,545 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:28:10,545 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:28:10,545 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:28:10,545 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:28:10,548 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:28:10,548 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:28:10,548 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:28:10,548 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:28:10,548 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:28:10,549 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:28:10,549 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:28:10,549 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:28:10,549 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:28:10,549 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:28:10,550 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:28:10,550 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:28:10,551 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:10,551 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:28:10,551 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:28:10,552 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:10,560 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:28:10,560 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:28:10,560 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:28:10,560 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:28:10,560 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:28:10,561 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:28:10,561 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:28:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-19T01:28:10,664 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357290664_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } 2023-11-19T01:28:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357290664 (current time:1700357290664). 2023-11-19T01:28:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:28:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357290664_ns2_test-17003571846791 VERSION not specified, setting to 2 2023-11-19T01:28:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a16bed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:10,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:10,674 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:35338, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:10,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:10,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:28:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=97, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1700357290664_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } 2023-11-19T01:28:10,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357290664_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 }, snapshot procedure id = 97 2023-11-19T01:28:10,677 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1700357290664_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:28:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=97 2023-11-19T01:28:10,678 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1700357290664_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:28:10,680 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357290664_ns2_test-17003571846791 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:28:10,683 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1700357290664_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:28:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=97 2023-11-19T01:28:10,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=97 2023-11-19T01:28:11,088 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1700357290664_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:28:11,088 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE; SnapshotRegionProcedure 4ef412d65669b2fe1684b7fdf6ee1f7d}] 2023-11-19T01:28:11,090 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=98, ppid=97, state=RUNNABLE; SnapshotRegionProcedure 4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:28:11,241 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:11,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=98 2023-11-19T01:28:11,242 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. 2023-11-19T01:28:11,242 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.HRegion(2520): Flush status journal for 4ef412d65669b2fe1684b7fdf6ee1f7d: 2023-11-19T01:28:11,242 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. for snapshot_1700357290664_ns2_test-17003571846791 completed. 2023-11-19T01:28:11,242 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(240): Storing 'ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d.' region-info for snapshot=snapshot_1700357290664_ns2_test-17003571846791 2023-11-19T01:28:11,242 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:28:11,242 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_] hfiles 2023-11-19T01:28:11,242 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_ for snapshot=snapshot_1700357290664_ns2_test-17003571846791 2023-11-19T01:28:11,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=97 2023-11-19T01:28:11,648 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. 2023-11-19T01:28:11,648 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=98 2023-11-19T01:28:11,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=98 2023-11-19T01:28:11,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357290664_ns2_test-17003571846791 on region 4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:28:11,650 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=98, ppid=97, state=RUNNABLE; SnapshotRegionProcedure 4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:28:11,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=98, resume processing ppid=97 2023-11-19T01:28:11,652 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1700357290664_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:28:11,652 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=98, ppid=97, state=SUCCESS; SnapshotRegionProcedure 4ef412d65669b2fe1684b7fdf6ee1f7d in 562 msec 2023-11-19T01:28:11,652 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1700357290664_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:28:11,653 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1700357290664_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:28:11,653 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357290664_ns2_test-17003571846791 2023-11-19T01:28:11,653 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357290664_ns2_test-17003571846791 2023-11-19T01:28:11,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=97 2023-11-19T01:28:12,062 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1700357290664_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:28:12,066 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1700357290664_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:28:12,067 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357290664_ns2_test-17003571846791 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357290664_ns2_test-17003571846791 2023-11-19T01:28:12,068 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=97, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1700357290664_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:28:12,068 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357290664_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 }, snapshot procedure id = 97 2023-11-19T01:28:12,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=97, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=97, snapshot={ ss=snapshot_1700357290664_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } in 1.3930 sec 2023-11-19T01:28:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=97 2023-11-19T01:28:12,784 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357292784_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } 2023-11-19T01:28:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357292784 (current time:1700357292784). 2023-11-19T01:28:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:28:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357292784_ns3_test-17003571846792 VERSION not specified, setting to 2 2023-11-19T01:28:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@281e49cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:12,794 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:35352, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:12,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:28:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=99, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1700357292784_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } 2023-11-19T01:28:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357292784_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 }, snapshot procedure id = 99 2023-11-19T01:28:12,799 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1700357292784_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:28:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=99 2023-11-19T01:28:12,800 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1700357292784_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:28:12,802 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357292784_ns3_test-17003571846792 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:28:12,805 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1700357292784_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:28:12,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=99 2023-11-19T01:28:13,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=99 2023-11-19T01:28:13,215 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1700357292784_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:28:13,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE; SnapshotRegionProcedure b78f59bec877d4d278f3912d6fb4631e}] 2023-11-19T01:28:13,217 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=100, ppid=99, state=RUNNABLE; SnapshotRegionProcedure b78f59bec877d4d278f3912d6fb4631e 2023-11-19T01:28:13,368 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:13,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=100 2023-11-19T01:28:13,369 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. 2023-11-19T01:28:13,369 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.HRegion(2520): Flush status journal for b78f59bec877d4d278f3912d6fb4631e: 2023-11-19T01:28:13,369 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. for snapshot_1700357292784_ns3_test-17003571846792 completed. 2023-11-19T01:28:13,369 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] snapshot.SnapshotManifest(240): Storing 'ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e.' region-info for snapshot=snapshot_1700357292784_ns3_test-17003571846792 2023-11-19T01:28:13,369 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:28:13,369 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-19T01:28:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=99 2023-11-19T01:28:13,775 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. 2023-11-19T01:28:13,775 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=100 2023-11-19T01:28:13,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=100 2023-11-19T01:28:13,775 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357292784_ns3_test-17003571846792 on region b78f59bec877d4d278f3912d6fb4631e 2023-11-19T01:28:13,777 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=100, ppid=99, state=RUNNABLE; SnapshotRegionProcedure b78f59bec877d4d278f3912d6fb4631e 2023-11-19T01:28:13,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=100, resume processing ppid=99 2023-11-19T01:28:13,779 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1700357292784_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:28:13,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=100, ppid=99, state=SUCCESS; SnapshotRegionProcedure b78f59bec877d4d278f3912d6fb4631e in 562 msec 2023-11-19T01:28:13,779 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1700357292784_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:28:13,779 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1700357292784_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:28:13,780 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357292784_ns3_test-17003571846792 2023-11-19T01:28:13,780 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357292784_ns3_test-17003571846792 2023-11-19T01:28:13,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=99 2023-11-19T01:28:14,190 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1700357292784_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:28:14,194 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1700357292784_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:28:14,195 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357292784_ns3_test-17003571846792 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357292784_ns3_test-17003571846792 2023-11-19T01:28:14,196 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=99, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1700357292784_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:28:14,196 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357292784_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 }, snapshot procedure id = 99 2023-11-19T01:28:14,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=99, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=99, snapshot={ ss=snapshot_1700357292784_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } in 1.4000 sec 2023-11-19T01:28:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=99 2023-11-19T01:28:14,904 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1700357286319 2023-11-19T01:28:14,904 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-19T01:28:14,904 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-11-19T01:28:14,904 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357286319_ns2:test-17003571846791 2023-11-19T01:28:14,904 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357290664_ns2_test-17003571846791 to hdfs://localhost:42739/backupUT/backup_1700357286319/ns2/test-17003571846791/ 2023-11-19T01:28:14,904 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:28:14,939 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:28:14,939 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,44889,1700357173572_1379169038_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42739/backupUT/backup_1700357286319/ns2/test-17003571846791, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42739/backupUT/backup_1700357286319/ns2/test-17003571846791/.hbase-snapshot/.tmp/snapshot_1700357290664_ns2_test-17003571846791 2023-11-19T01:28:14,940 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af, snapshotDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357290664_ns2_test-17003571846791. 2023-11-19T01:28:14,948 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357290664_ns2_test-17003571846791 to hdfs://localhost:42739/backupUT/backup_1700357286319/ns2/test-17003571846791/.hbase-snapshot/.tmp/snapshot_1700357290664_ns2_test-17003571846791 2023-11-19T01:28:15,366 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:15,367 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:15,367 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:15,367 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:16,380 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-6692840527727030365.jar 2023-11-19T01:28:16,381 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:16,381 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:16,382 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:16,382 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:16,382 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:16,383 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:16,383 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:16,383 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:16,384 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:16,384 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:28:16,385 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:28:16,385 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:28:16,386 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:28:16,386 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:28:16,387 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:28:16,387 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:28:16,387 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:28:16,388 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:28:16,388 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:28:16,389 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:28:16,389 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:28:16,390 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:16,390 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:16,391 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:16,391 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:16,391 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:16,392 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:16,392 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:16,435 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357290664_ns2_test-17003571846791' hfile list 2023-11-19T01:28:16,438 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-19T01:28:16,511 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357296476/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:28:16,511 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357296476/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:28:16,565 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-19T01:28:16,588 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_ output=hdfs://localhost:42739/backupUT/backup_1700357286319/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_ 2023-11-19T01:28:16,588 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 8.321M/sec 2023-11-19T01:28:17,544 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:28:17,545 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:28:17,553 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357290664_ns2_test-17003571846791 2023-11-19T01:28:17,554 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357290664_ns2_test-17003571846791 finished. 2023-11-19T01:28:17,554 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357286319_ns3:test-17003571846792 2023-11-19T01:28:17,554 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357292784_ns3_test-17003571846792 to hdfs://localhost:42739/backupUT/backup_1700357286319/ns3/test-17003571846792/ 2023-11-19T01:28:17,554 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:28:17,579 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:28:17,579 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,44889,1700357173572_-2103487586_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42739/backupUT/backup_1700357286319/ns3/test-17003571846792, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42739/backupUT/backup_1700357286319/ns3/test-17003571846792/.hbase-snapshot/.tmp/snapshot_1700357292784_ns3_test-17003571846792 2023-11-19T01:28:17,579 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af, snapshotDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357292784_ns3_test-17003571846792. 2023-11-19T01:28:17,587 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357292784_ns3_test-17003571846792 to hdfs://localhost:42739/backupUT/backup_1700357286319/ns3/test-17003571846792/.hbase-snapshot/.tmp/snapshot_1700357292784_ns3_test-17003571846792 2023-11-19T01:28:18,002 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:18,002 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:18,003 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:18,003 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:18,504 DEBUG [regionserver/jenkins-hbase6:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.38461538461538464, tune throughput to 69.23 MB/second 2023-11-19T01:28:19,016 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-1546324832246077980.jar 2023-11-19T01:28:19,016 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:19,017 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:19,017 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:19,018 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:19,018 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:19,018 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:19,019 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:19,019 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:19,019 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:19,020 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:28:19,021 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:28:19,021 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:28:19,021 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:28:19,022 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:28:19,022 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:28:19,023 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:28:19,023 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:28:19,023 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:28:19,024 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:28:19,024 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:28:19,024 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:28:19,025 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:19,025 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:19,026 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:19,026 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:19,027 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:19,027 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:19,028 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:19,071 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357292784_ns3_test-17003571846792' hfile list 2023-11-19T01:28:19,140 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357299107/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:28:19,140 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357299107/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:28:20,167 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:28:20,168 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:28:20,173 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357292784_ns3_test-17003571846792 2023-11-19T01:28:20,173 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357292784_ns3_test-17003571846792 finished. 2023-11-19T01:28:20,173 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1034): ns2:test-17003571846791 2023-11-19T01:28:20,173 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1034): ns3:test-17003571846792 2023-11-19T01:28:20,179 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357286319 2023-11-19T01:28:20,179 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:28:20,584 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42739/backupUT/backup_1700357286319/.backup.manifest 2023-11-19T01:28:20,584 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357286319 2023-11-19T01:28:20,584 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:28:20,990 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42739/backupUT/backup_1700357286319/.backup.manifest 2023-11-19T01:28:20,990 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(358): Backup backup_1700357286319 finished: type=FULL,tablelist=ns2:test-17003571846791;ns3:test-17003571846792,targetRootDir=hdfs://localhost:42739/backupUT,startts=1700357290471,completets=1700357300179,bytescopied=0 2023-11-19T01:28:20,990 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-19T01:28:20,990 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357290664_ns2_test-17003571846791 2023-11-19T01:28:20,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357290664_ns2_test-17003571846791" 2023-11-19T01:28:20,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357290664_ns2_test-17003571846791 2023-11-19T01:28:20,994 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357290664_ns2_test-17003571846791 for backup backup_1700357286319 succeeded. 2023-11-19T01:28:20,994 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357292784_ns3_test-17003571846792 2023-11-19T01:28:20,994 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357292784_ns3_test-17003571846792" 2023-11-19T01:28:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357292784_ns3_test-17003571846792 2023-11-19T01:28:20,997 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357292784_ns3_test-17003571846792 for backup backup_1700357286319 succeeded. 2023-11-19T01:28:20,998 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:28:21,002 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:28:21,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:28:21,005 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:28:21,006 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:28:21,012 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(376): Backup backup_1700357286319 completed. 2023-11-19T01:28:21,013 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:21,024 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(101): HBase table ns2:table2 does not exist. It will be created during restore process 2023-11-19T01:28:21,025 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(101): HBase table ns3:table3_restore does not exist. It will be created during restore process 2023-11-19T01:28:21,026 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357286319/.backup.manifest 2023-11-19T01:28:21,028 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357286319/.backup.manifest 2023-11-19T01:28:21,028 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357286319/.backup.manifest 2023-11-19T01:28:21,030 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357286319/.backup.manifest 2023-11-19T01:28:21,030 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357286319/.backup.manifest 2023-11-19T01:28:21,032 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357286319/.backup.manifest 2023-11-19T01:28:21,032 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-17003571846791' to 'ns2:table2' from full backup image hdfs://localhost:42739/backupUT/backup_1700357286319/ns2/test-17003571846791 2023-11-19T01:28:21,038 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(487): Creating target table 'ns2:table2' 2023-11-19T01:28:21,038 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:42739/backupUT/backup_1700357286319/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:28:21,039 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:42739/backupUT/backup_1700357286319/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f in region [hdfs://localhost:42739/backupUT/backup_1700357286319/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d] 2023-11-19T01:28:21,042 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:42739/backupUT/backup_1700357286319/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_ first=row0 last=row98 2023-11-19T01:28:21,043 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:28:21,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=101, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:table2 2023-11-19T01:28:21,046 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:28:21,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "ns2" qualifier: "table2" procId is: 101 2023-11-19T01:28:21,047 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:28:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=101 2023-11-19T01:28:21,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=101 2023-11-19T01:28:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=101 2023-11-19T01:28:21,454 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => db93ec3357b67e54199e6a10477c986c, NAME => 'ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:28:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=101 2023-11-19T01:28:21,868 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:28:21,868 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1668): Closing db93ec3357b67e54199e6a10477c986c, disabling compactions & flushes 2023-11-19T01:28:21,868 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c. 2023-11-19T01:28:21,868 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c. 2023-11-19T01:28:21,868 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c. after waiting 0 ms 2023-11-19T01:28:21,868 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c. 2023-11-19T01:28:21,868 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1909): Closed ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c. 2023-11-19T01:28:21,868 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1622): Region close journal for db93ec3357b67e54199e6a10477c986c: 2023-11-19T01:28:21,869 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:28:21,869 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1700357301869"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357301869"}]},"ts":"1700357301869"} 2023-11-19T01:28:21,871 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:28:21,875 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:28:21,875 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357301875"}]},"ts":"1700357301875"} 2023-11-19T01:28:21,876 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2023-11-19T01:28:21,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=db93ec3357b67e54199e6a10477c986c, ASSIGN}] 2023-11-19T01:28:21,884 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=db93ec3357b67e54199e6a10477c986c, ASSIGN 2023-11-19T01:28:21,885 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=102, ppid=101, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:table2, region=db93ec3357b67e54199e6a10477c986c, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,44997,1700357176784; forceNewPlan=false, retain=false 2023-11-19T01:28:22,036 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=102 updating hbase:meta row=db93ec3357b67e54199e6a10477c986c, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:22,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE; OpenRegionProcedure db93ec3357b67e54199e6a10477c986c, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:28:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=101 2023-11-19T01:28:22,189 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:22,192 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] handler.AssignRegionHandler(130): Open ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c. 2023-11-19T01:28:22,192 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7254): Opening region: {ENCODED => db93ec3357b67e54199e6a10477c986c, NAME => 'ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:28:22,193 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:28:22,193 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 db93ec3357b67e54199e6a10477c986c 2023-11-19T01:28:22,193 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(888): Instantiated ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:28:22,193 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7296): checking encryption for db93ec3357b67e54199e6a10477c986c 2023-11-19T01:28:22,193 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(7299): checking classloading for db93ec3357b67e54199e6a10477c986c 2023-11-19T01:28:22,194 INFO [StoreOpener-db93ec3357b67e54199e6a10477c986c-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region db93ec3357b67e54199e6a10477c986c 2023-11-19T01:28:22,196 INFO [StoreOpener-db93ec3357b67e54199e6a10477c986c-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db93ec3357b67e54199e6a10477c986c columnFamilyName f 2023-11-19T01:28:22,196 DEBUG [StoreOpener-db93ec3357b67e54199e6a10477c986c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:28:22,196 INFO [StoreOpener-db93ec3357b67e54199e6a10477c986c-1 {}] regionserver.HStore(324): Store=db93ec3357b67e54199e6a10477c986c/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:28:22,197 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/db93ec3357b67e54199e6a10477c986c 2023-11-19T01:28:22,197 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/db93ec3357b67e54199e6a10477c986c 2023-11-19T01:28:22,199 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1076): writing seq id for db93ec3357b67e54199e6a10477c986c 2023-11-19T01:28:22,201 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/db93ec3357b67e54199e6a10477c986c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:28:22,201 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(1093): Opened db93ec3357b67e54199e6a10477c986c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11610594080, jitterRate=0.08132083714008331}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:28:22,202 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegion(992): Region open journal for db93ec3357b67e54199e6a10477c986c: 2023-11-19T01:28:22,203 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c., pid=103, masterSystemTime=1700357302189 2023-11-19T01:28:22,204 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c. 2023-11-19T01:28:22,204 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=103}] handler.AssignRegionHandler(158): Opened ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c. 2023-11-19T01:28:22,204 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=102 updating hbase:meta row=db93ec3357b67e54199e6a10477c986c, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:22,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=103, resume processing ppid=102 2023-11-19T01:28:22,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=103, ppid=102, state=SUCCESS; OpenRegionProcedure db93ec3357b67e54199e6a10477c986c, server=jenkins-hbase6.apache.org,44997,1700357176784 in 168 msec 2023-11-19T01:28:22,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=102, resume processing ppid=101 2023-11-19T01:28:22,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=102, ppid=101, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=db93ec3357b67e54199e6a10477c986c, ASSIGN in 325 msec 2023-11-19T01:28:22,209 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:28:22,209 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357302209"}]},"ts":"1700357302209"} 2023-11-19T01:28:22,210 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2023-11-19T01:28:22,212 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=101, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:28:22,213 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=101, state=SUCCESS; CreateTableProcedure table=ns2:table2 in 1.1690 sec 2023-11-19T01:28:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=101 2023-11-19T01:28:23,155 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:42739/backupUT/backup_1700357286319/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d from hbase tables ns2:test-17003571846791 to tables ns2:table2 2023-11-19T01:28:23,155 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-17003571846791 into ns2:table2 2023-11-19T01:28:23,156 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-table2-1700357303155 from hdfs://localhost:42739/backupUT/backup_1700357286319/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:28:23,163 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3edfbcc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:23,163 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:28:23,164 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:49096, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:28:23,165 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-19T01:28:23,165 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:table2 2023-11-19T01:28:23,168 DEBUG [hconnection-0x504e91a6-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:23,169 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44296, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:23,170 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:table2: 2023-11-19T01:28:23,170 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-11-19T01:28:23,171 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/partitions_3251d23c-3920-4ace-b2d0-3ba356b216f0 2023-11-19T01:28:23,599 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:23,599 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:23,600 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:23,600 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:24,632 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-5429812268421568025.jar 2023-11-19T01:28:24,632 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:24,632 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:24,633 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:24,633 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:24,634 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:24,634 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:24,634 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:24,635 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:24,635 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:24,636 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:28:24,636 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:28:24,637 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:28:24,637 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:28:24,637 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:28:24,638 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:28:24,638 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:28:24,639 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:28:24,639 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:28:24,640 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:28:24,641 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:28:24,641 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:28:24,641 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:24,642 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:24,642 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:24,642 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:24,643 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:24,643 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:24,644 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:24,644 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:table2 2023-11-19T01:28:24,644 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:24,644 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-11-19T01:28:24,645 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-11-19T01:28:24,646 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:28:24,658 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-11-19T01:28:24,742 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357304708/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:28:24,742 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357304708/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:28:24,784 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2023-11-19T01:28:24,784 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_table2 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:28:24,791 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:42739/backupUT/backup_1700357286319/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_ 2023-11-19T01:28:24,795 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-11-19T01:28:24,817 DEBUG [pool-397-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7007aa41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:24,822 DEBUG [hconnection-0x69dde96c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:24,824 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44298, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:24,825 DEBUG [pool-397-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:24,826 DEBUG [pool-397-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-11-19T01:28:24,826 DEBUG [pool-397-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase6.apache.org 2023-11-19T01:28:24,834 DEBUG [pool-397-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-table2-1700357303155/_temporary/0/_temporary/attempt_local1572031507_0016_r_000000_0/f/4a8eeafd3eb140d4bb36792dd5ad7855 is 37, key is row10/f:q1/1700357188531/Put/seqid=0 2023-11-19T01:28:24,835 WARN [Thread-1715 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase6.apache.org:44997] Specified favored nodes: [jenkins-hbase6.apache.org:44997] 2023-11-19T01:28:25,485 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns2:table2' 2023-11-19T01:28:25,794 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-table2-1700357303155 2023-11-19T01:28:25,800 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b27694d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:25,802 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:25,804 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44306, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:25,809 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-table2-1700357303155/_SUCCESS 2023-11-19T01:28:25,812 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:28:25,814 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:49100, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:28:25,822 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-table2-1700357303155/f/4a8eeafd3eb140d4bb36792dd5ad7855 first=Optional[row0] last=Optional[row98] 2023-11-19T01:28:25,823 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c., hostname=jenkins-hbase6.apache.org,44997,1700357176784, seqNum=2 for row with hfile group [{f,hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-table2-1700357303155/f/4a8eeafd3eb140d4bb36792dd5ad7855}] 2023-11-19T01:28:25,829 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-table2-1700357303155/f/4a8eeafd3eb140d4bb36792dd5ad7855 for inclusion in db93ec3357b67e54199e6a10477c986c/f 2023-11-19T01:28:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-11-19T01:28:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(625): Region bounds: first= last= 2023-11-19T01:28:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegion(2520): Flush status journal for db93ec3357b67e54199e6a10477c986c: 2023-11-19T01:28:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-table2-1700357303155/f/4a8eeafd3eb140d4bb36792dd5ad7855 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__ns2_table2__8dpo70lerl0qrg4d4l524pa8qemdancmvnj9piem8d2d41p3gjf18jmhinqsjm38/f/4a8eeafd3eb140d4bb36792dd5ad7855 2023-11-19T01:28:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__ns2_table2__8dpo70lerl0qrg4d4l524pa8qemdancmvnj9piem8d2d41p3gjf18jmhinqsjm38/f/4a8eeafd3eb140d4bb36792dd5ad7855 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/db93ec3357b67e54199e6a10477c986c/f/945c0bcbb8db4a93ac63e8acdd5bae3b_SeqId_4_ 2023-11-19T01:28:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4799e915, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:28:25,847 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:49106, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:28:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:25,852 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44312, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:25,865 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__ns2_table2__8dpo70lerl0qrg4d4l524pa8qemdancmvnj9piem8d2d41p3gjf18jmhinqsjm38/f/4a8eeafd3eb140d4bb36792dd5ad7855 into db93ec3357b67e54199e6a10477c986c/f as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/db93ec3357b67e54199e6a10477c986c/f/945c0bcbb8db4a93ac63e8acdd5bae3b_SeqId_4_ - updating store file list. 2023-11-19T01:28:25,871 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/db93ec3357b67e54199e6a10477c986c/f/945c0bcbb8db4a93ac63e8acdd5bae3b_SeqId_4_ into db93ec3357b67e54199e6a10477c986c/f 2023-11-19T01:28:25,871 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__ns2_table2__8dpo70lerl0qrg4d4l524pa8qemdancmvnj9piem8d2d41p3gjf18jmhinqsjm38/f/4a8eeafd3eb140d4bb36792dd5ad7855 into db93ec3357b67e54199e6a10477c986c/f (new location: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/db93ec3357b67e54199e6a10477c986c/f/945c0bcbb8db4a93ac63e8acdd5bae3b_SeqId_4_) 2023-11-19T01:28:25,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__ns2_table2__8dpo70lerl0qrg4d4l524pa8qemdancmvnj9piem8d2d41p3gjf18jmhinqsjm38/f/4a8eeafd3eb140d4bb36792dd5ad7855 2023-11-19T01:28:25,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33d79af8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:25,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:28:25,881 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:49110, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-19T01:28:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:25,886 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44328, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-11-19T01:28:25,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:25,901 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:25,901 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-11-19T01:28:25,901 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-19T01:28:25,901 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(233): Backup: backup_1700357286319 hdfs://localhost:42739/backupUT/backup_1700357286319/ns2/test-17003571846791/ 2023-11-19T01:28:25,902 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357286319/.backup.manifest 2023-11-19T01:28:25,906 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357286319/.backup.manifest 2023-11-19T01:28:25,906 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-17003571846792' to 'ns3:table3_restore' from full backup image hdfs://localhost:42739/backupUT/backup_1700357286319/ns3/test-17003571846792 2023-11-19T01:28:25,912 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:42739/backupUT/backup_1700357286319/ns3/test-17003571846792/archive/data/ns3/test-17003571846792 does not exists 2023-11-19T01:28:25,912 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-17003571846792, will only create table 2023-11-19T01:28:25,913 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(487): Creating target table 'ns3:table3_restore' 2023-11-19T01:28:25,914 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:28:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:table3_restore 2023-11-19T01:28:25,916 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:28:25,917 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "ns3" qualifier: "table3_restore" procId is: 104 2023-11-19T01:28:25,917 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:28:25,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=104 2023-11-19T01:28:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=104 2023-11-19T01:28:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=104 2023-11-19T01:28:26,324 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => ac6a1d5ffef5e5d59fc8adf5d1615b43, NAME => 'ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:28:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=104 2023-11-19T01:28:26,729 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:28:26,730 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1668): Closing ac6a1d5ffef5e5d59fc8adf5d1615b43, disabling compactions & flushes 2023-11-19T01:28:26,730 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43. 2023-11-19T01:28:26,730 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43. 2023-11-19T01:28:26,730 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43. after waiting 0 ms 2023-11-19T01:28:26,730 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43. 2023-11-19T01:28:26,730 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43. 2023-11-19T01:28:26,730 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1622): Region close journal for ac6a1d5ffef5e5d59fc8adf5d1615b43: 2023-11-19T01:28:26,731 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:28:26,731 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1700357306731"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357306731"}]},"ts":"1700357306731"} 2023-11-19T01:28:26,732 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:28:26,733 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:28:26,733 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357306733"}]},"ts":"1700357306733"} 2023-11-19T01:28:26,734 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLING in hbase:meta 2023-11-19T01:28:26,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=ac6a1d5ffef5e5d59fc8adf5d1615b43, ASSIGN}] 2023-11-19T01:28:26,746 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=ac6a1d5ffef5e5d59fc8adf5d1615b43, ASSIGN 2023-11-19T01:28:26,747 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:table3_restore, region=ac6a1d5ffef5e5d59fc8adf5d1615b43, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,44997,1700357176784; forceNewPlan=false, retain=false 2023-11-19T01:28:26,898 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=105 updating hbase:meta row=ac6a1d5ffef5e5d59fc8adf5d1615b43, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:26,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE; OpenRegionProcedure ac6a1d5ffef5e5d59fc8adf5d1615b43, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:28:27,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=104 2023-11-19T01:28:27,051 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:27,054 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(130): Open ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43. 2023-11-19T01:28:27,054 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7254): Opening region: {ENCODED => ac6a1d5ffef5e5d59fc8adf5d1615b43, NAME => 'ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:28:27,054 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:28:27,054 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table3_restore ac6a1d5ffef5e5d59fc8adf5d1615b43 2023-11-19T01:28:27,055 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:28:27,055 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7296): checking encryption for ac6a1d5ffef5e5d59fc8adf5d1615b43 2023-11-19T01:28:27,055 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7299): checking classloading for ac6a1d5ffef5e5d59fc8adf5d1615b43 2023-11-19T01:28:27,056 INFO [StoreOpener-ac6a1d5ffef5e5d59fc8adf5d1615b43-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region ac6a1d5ffef5e5d59fc8adf5d1615b43 2023-11-19T01:28:27,057 INFO [StoreOpener-ac6a1d5ffef5e5d59fc8adf5d1615b43-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac6a1d5ffef5e5d59fc8adf5d1615b43 columnFamilyName f 2023-11-19T01:28:27,057 DEBUG [StoreOpener-ac6a1d5ffef5e5d59fc8adf5d1615b43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:28:27,057 INFO [StoreOpener-ac6a1d5ffef5e5d59fc8adf5d1615b43-1 {}] regionserver.HStore(324): Store=ac6a1d5ffef5e5d59fc8adf5d1615b43/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:28:27,058 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/table3_restore/ac6a1d5ffef5e5d59fc8adf5d1615b43 2023-11-19T01:28:27,058 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/table3_restore/ac6a1d5ffef5e5d59fc8adf5d1615b43 2023-11-19T01:28:27,060 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1076): writing seq id for ac6a1d5ffef5e5d59fc8adf5d1615b43 2023-11-19T01:28:27,062 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/table3_restore/ac6a1d5ffef5e5d59fc8adf5d1615b43/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:28:27,062 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1093): Opened ac6a1d5ffef5e5d59fc8adf5d1615b43; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=12009640640, jitterRate=0.11848494410514832}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:28:27,063 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(992): Region open journal for ac6a1d5ffef5e5d59fc8adf5d1615b43: 2023-11-19T01:28:27,063 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43., pid=106, masterSystemTime=1700357307051 2023-11-19T01:28:27,065 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43. 2023-11-19T01:28:27,065 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(158): Opened ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43. 2023-11-19T01:28:27,065 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=105 updating hbase:meta row=ac6a1d5ffef5e5d59fc8adf5d1615b43, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:27,067 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=106, resume processing ppid=105 2023-11-19T01:28:27,067 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=106, ppid=105, state=SUCCESS; OpenRegionProcedure ac6a1d5ffef5e5d59fc8adf5d1615b43, server=jenkins-hbase6.apache.org,44997,1700357176784 in 166 msec 2023-11-19T01:28:27,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=105, resume processing ppid=104 2023-11-19T01:28:27,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=105, ppid=104, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=ac6a1d5ffef5e5d59fc8adf5d1615b43, ASSIGN in 325 msec 2023-11-19T01:28:27,069 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:28:27,069 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357307069"}]},"ts":"1700357307069"} 2023-11-19T01:28:27,070 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLED in hbase:meta 2023-11-19T01:28:27,072 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=104, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:28:27,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=104, state=SUCCESS; CreateTableProcedure table=ns3:table3_restore in 1.1580 sec 2023-11-19T01:28:27,734 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:28:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=104 2023-11-19T01:28:28,025 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-19T01:28:28,026 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(233): Backup: backup_1700357286319 hdfs://localhost:42739/backupUT/backup_1700357286319/ns3/test-17003571846792/ 2023-11-19T01:28:28,026 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-11-19T01:28:28,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable ns2:table2 2023-11-19T01:28:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:table2 2023-11-19T01:28:28,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=107 2023-11-19T01:28:28,031 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357308031"}]},"ts":"1700357308031"} 2023-11-19T01:28:28,033 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLING in hbase:meta 2023-11-19T01:28:28,035 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set ns2:table2 to state=DISABLING 2023-11-19T01:28:28,035 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=db93ec3357b67e54199e6a10477c986c, UNASSIGN}] 2023-11-19T01:28:28,037 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=108, ppid=107, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=db93ec3357b67e54199e6a10477c986c, UNASSIGN 2023-11-19T01:28:28,038 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=108 updating hbase:meta row=db93ec3357b67e54199e6a10477c986c, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:28,039 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE; CloseRegionProcedure db93ec3357b67e54199e6a10477c986c, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:28:28,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=107 2023-11-19T01:28:28,189 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:28,189 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] handler.UnassignRegionHandler(119): Close db93ec3357b67e54199e6a10477c986c 2023-11-19T01:28:28,189 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1668): Closing db93ec3357b67e54199e6a10477c986c, disabling compactions & flushes 2023-11-19T01:28:28,189 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1690): Closing region ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c. 2023-11-19T01:28:28,189 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c. 2023-11-19T01:28:28,190 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c. after waiting 0 ms 2023-11-19T01:28:28,190 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c. 2023-11-19T01:28:28,195 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/db93ec3357b67e54199e6a10477c986c/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-11-19T01:28:28,196 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:28:28,197 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1909): Closed ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c. 2023-11-19T01:28:28,197 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] regionserver.HRegion(1622): Region close journal for db93ec3357b67e54199e6a10477c986c: 2023-11-19T01:28:28,198 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=109}] handler.UnassignRegionHandler(163): Closed db93ec3357b67e54199e6a10477c986c 2023-11-19T01:28:28,198 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=108 updating hbase:meta row=db93ec3357b67e54199e6a10477c986c, regionState=CLOSED 2023-11-19T01:28:28,201 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=109, resume processing ppid=108 2023-11-19T01:28:28,201 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=109, ppid=108, state=SUCCESS; CloseRegionProcedure db93ec3357b67e54199e6a10477c986c, server=jenkins-hbase6.apache.org,44997,1700357176784 in 161 msec 2023-11-19T01:28:28,202 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=108, resume processing ppid=107 2023-11-19T01:28:28,202 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=108, ppid=107, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=db93ec3357b67e54199e6a10477c986c, UNASSIGN in 166 msec 2023-11-19T01:28:28,203 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357308203"}]},"ts":"1700357308203"} 2023-11-19T01:28:28,204 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLED in hbase:meta 2023-11-19T01:28:28,207 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set ns2:table2 to state=DISABLED 2023-11-19T01:28:28,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=107, state=SUCCESS; DisableTableProcedure table=ns2:table2 in 179 msec 2023-11-19T01:28:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=107 2023-11-19T01:28:28,334 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$5(2449): Client=jenkins//172.31.2.20 delete ns2:table2 2023-11-19T01:28:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=110, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns2:table2 2023-11-19T01:28:28,337 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=110, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-11-19T01:28:28,338 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=110, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns2:table2 2023-11-19T01:28:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=110 2023-11-19T01:28:28,342 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/db93ec3357b67e54199e6a10477c986c 2023-11-19T01:28:28,344 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/db93ec3357b67e54199e6a10477c986c/f, FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/db93ec3357b67e54199e6a10477c986c/recovered.edits] 2023-11-19T01:28:28,349 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/db93ec3357b67e54199e6a10477c986c/f/945c0bcbb8db4a93ac63e8acdd5bae3b_SeqId_4_ to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/archive/data/ns2/table2/db93ec3357b67e54199e6a10477c986c/f/945c0bcbb8db4a93ac63e8acdd5bae3b_SeqId_4_ 2023-11-19T01:28:28,352 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/db93ec3357b67e54199e6a10477c986c/recovered.edits/6.seqid to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/archive/data/ns2/table2/db93ec3357b67e54199e6a10477c986c/recovered.edits/6.seqid 2023-11-19T01:28:28,352 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/db93ec3357b67e54199e6a10477c986c 2023-11-19T01:28:28,352 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(293): Archived ns2:table2 regions 2023-11-19T01:28:28,354 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=110, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns2:table2 2023-11-19T01:28:28,356 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:table2 from hbase:meta 2023-11-19T01:28:28,358 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:table2' descriptor. 2023-11-19T01:28:28,359 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=110, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns2:table2 2023-11-19T01:28:28,359 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:table2' from region states. 2023-11-19T01:28:28,359 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357308359"}]},"ts":"9223372036854775807"} 2023-11-19T01:28:28,363 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:28:28,363 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => db93ec3357b67e54199e6a10477c986c, NAME => 'ns2:table2,,1700357301043.db93ec3357b67e54199e6a10477c986c.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:28:28,363 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:table2' as deleted. 2023-11-19T01:28:28,363 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1700357308363"}]},"ts":"9223372036854775807"} 2023-11-19T01:28:28,364 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:table2 state from META 2023-11-19T01:28:28,366 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(127): Finished pid=110, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-11-19T01:28:28,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=110, state=SUCCESS; DeleteTableProcedure table=ns2:table2 in 32 msec 2023-11-19T01:28:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=110 2023-11-19T01:28:28,440 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable ns3:table3_restore 2023-11-19T01:28:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:table3_restore 2023-11-19T01:28:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=111 2023-11-19T01:28:28,444 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357308443"}]},"ts":"1700357308443"} 2023-11-19T01:28:28,445 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLING in hbase:meta 2023-11-19T01:28:28,446 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(293): Set ns3:table3_restore to state=DISABLING 2023-11-19T01:28:28,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=ac6a1d5ffef5e5d59fc8adf5d1615b43, UNASSIGN}] 2023-11-19T01:28:28,449 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=112, ppid=111, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=ac6a1d5ffef5e5d59fc8adf5d1615b43, UNASSIGN 2023-11-19T01:28:28,449 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=112 updating hbase:meta row=ac6a1d5ffef5e5d59fc8adf5d1615b43, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:28,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE; CloseRegionProcedure ac6a1d5ffef5e5d59fc8adf5d1615b43, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:28:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=111 2023-11-19T01:28:28,601 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:28,602 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] handler.UnassignRegionHandler(119): Close ac6a1d5ffef5e5d59fc8adf5d1615b43 2023-11-19T01:28:28,603 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1668): Closing ac6a1d5ffef5e5d59fc8adf5d1615b43, disabling compactions & flushes 2023-11-19T01:28:28,603 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43. 2023-11-19T01:28:28,603 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43. 2023-11-19T01:28:28,603 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43. after waiting 0 ms 2023-11-19T01:28:28,603 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43. 2023-11-19T01:28:28,608 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/table3_restore/ac6a1d5ffef5e5d59fc8adf5d1615b43/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-19T01:28:28,609 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:28:28,609 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43. 2023-11-19T01:28:28,609 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] regionserver.HRegion(1622): Region close journal for ac6a1d5ffef5e5d59fc8adf5d1615b43: 2023-11-19T01:28:28,610 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=113}] handler.UnassignRegionHandler(163): Closed ac6a1d5ffef5e5d59fc8adf5d1615b43 2023-11-19T01:28:28,611 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=112 updating hbase:meta row=ac6a1d5ffef5e5d59fc8adf5d1615b43, regionState=CLOSED 2023-11-19T01:28:28,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=113, resume processing ppid=112 2023-11-19T01:28:28,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=113, ppid=112, state=SUCCESS; CloseRegionProcedure ac6a1d5ffef5e5d59fc8adf5d1615b43, server=jenkins-hbase6.apache.org,44997,1700357176784 in 162 msec 2023-11-19T01:28:28,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=112, resume processing ppid=111 2023-11-19T01:28:28,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=112, ppid=111, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=ac6a1d5ffef5e5d59fc8adf5d1615b43, UNASSIGN in 166 msec 2023-11-19T01:28:28,615 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357308615"}]},"ts":"1700357308615"} 2023-11-19T01:28:28,616 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLED in hbase:meta 2023-11-19T01:28:28,617 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(305): Set ns3:table3_restore to state=DISABLED 2023-11-19T01:28:28,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=111, state=SUCCESS; DisableTableProcedure table=ns3:table3_restore in 178 msec 2023-11-19T01:28:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=111 2023-11-19T01:28:28,746 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$5(2449): Client=jenkins//172.31.2.20 delete ns3:table3_restore 2023-11-19T01:28:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=114, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns3:table3_restore 2023-11-19T01:28:28,748 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=114, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-11-19T01:28:28,749 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=114, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-11-19T01:28:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=114 2023-11-19T01:28:28,751 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/table3_restore/ac6a1d5ffef5e5d59fc8adf5d1615b43 2023-11-19T01:28:28,754 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/table3_restore/ac6a1d5ffef5e5d59fc8adf5d1615b43/f, FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/table3_restore/ac6a1d5ffef5e5d59fc8adf5d1615b43/recovered.edits] 2023-11-19T01:28:28,758 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/table3_restore/ac6a1d5ffef5e5d59fc8adf5d1615b43/recovered.edits/4.seqid to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/archive/data/ns3/table3_restore/ac6a1d5ffef5e5d59fc8adf5d1615b43/recovered.edits/4.seqid 2023-11-19T01:28:28,759 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/table3_restore/ac6a1d5ffef5e5d59fc8adf5d1615b43 2023-11-19T01:28:28,759 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(293): Archived ns3:table3_restore regions 2023-11-19T01:28:28,762 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=114, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-11-19T01:28:28,763 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:table3_restore from hbase:meta 2023-11-19T01:28:28,765 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:table3_restore' descriptor. 2023-11-19T01:28:28,766 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=114, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-11-19T01:28:28,766 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:table3_restore' from region states. 2023-11-19T01:28:28,766 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357308766"}]},"ts":"9223372036854775807"} 2023-11-19T01:28:28,768 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:28:28,768 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => ac6a1d5ffef5e5d59fc8adf5d1615b43, NAME => 'ns3:table3_restore,,1700357305914.ac6a1d5ffef5e5d59fc8adf5d1615b43.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:28:28,768 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:table3_restore' as deleted. 2023-11-19T01:28:28,768 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1700357308768"}]},"ts":"9223372036854775807"} 2023-11-19T01:28:28,769 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:table3_restore state from META 2023-11-19T01:28:28,772 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(127): Finished pid=114, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-11-19T01:28:28,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=114, state=SUCCESS; DeleteTableProcedure table=ns3:table3_restore in 26 msec 2023-11-19T01:28:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=114 2023-11-19T01:28:28,871 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultiple Thread=602 (was 598) - Thread LEAK? -, OpenFileDescriptor=704 (was 711), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=114 (was 98) - SystemLoadAverage LEAK? -, ProcessCount=167 (was 167), AvailableMemoryMB=3530 (was 3602) 2023-11-19T01:28:28,871 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=602 is superior to 500 2023-11-19T01:28:28,891 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleOverwriteCommand Thread=602, OpenFileDescriptor=704, MaxFileDescriptor=60000, SystemLoadAverage=114, ProcessCount=167, AvailableMemoryMB=3529 2023-11-19T01:28:28,891 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=602 is superior to 500 2023-11-19T01:28:28,891 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestFullRestore(234): create full backup image on multiple tables: command-line 2023-11-19T01:28:28,898 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c784461, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:28,902 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:28,904 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44340, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:28,907 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:28:28,908 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:49124, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:28:28,919 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:28:28,924 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:28:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357308924 (current time:1700357308924). 2023-11-19T01:28:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:28:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:28:28,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f0e9935, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:28,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:28,934 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:44354, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:28,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:28,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:28:28,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=115, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:28:28,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 115 2023-11-19T01:28:28,940 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:28:28,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=115 2023-11-19T01:28:28,941 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:28:28,947 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:28:28,951 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:28:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=115 2023-11-19T01:28:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=115 2023-11-19T01:28:29,356 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:28:29,359 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc}] 2023-11-19T01:28:29,361 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=116, ppid=115, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:28:29,512 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:29,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=116 2023-11-19T01:28:29,513 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:28:29,513 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegion(2819): Flushing 322466352cac3ade468ab2ce8e706acc 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-11-19T01:28:29,516 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/a356c57983bf487eaa945a53843b801e is 171, key is trslm:hdfs://localhost:42739/backupUT\x00ns2:test-17003571846791/meta:log-roll-map/1700357300175/Put/seqid=0 2023-11-19T01:28:29,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=115 2023-11-19T01:28:29,921 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/a356c57983bf487eaa945a53843b801e 2023-11-19T01:28:29,929 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/4b8411bfe401452da4354337a869a577 is 524, key is session:backup_1700357286319/session:context/1700357301005/Put/seqid=0 2023-11-19T01:28:29,933 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/4b8411bfe401452da4354337a869a577 2023-11-19T01:28:29,938 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/a356c57983bf487eaa945a53843b801e as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/a356c57983bf487eaa945a53843b801e 2023-11-19T01:28:29,943 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/a356c57983bf487eaa945a53843b801e, entries=6, sequenceid=94, filesize=5.8 K 2023-11-19T01:28:29,944 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/4b8411bfe401452da4354337a869a577 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/4b8411bfe401452da4354337a869a577 2023-11-19T01:28:29,949 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/4b8411bfe401452da4354337a869a577, entries=2, sequenceid=94, filesize=5.5 K 2023-11-19T01:28:29,949 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for 322466352cac3ade468ab2ce8e706acc in 436ms, sequenceid=94, compaction requested=true 2023-11-19T01:28:29,950 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.HRegion(2520): Flush status journal for 322466352cac3ade468ab2ce8e706acc: 2023-11-19T01:28:29,950 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. for snapshot_backup_system completed. 2023-11-19T01:28:29,950 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:28:29,950 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:28:29,950 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/83b8d86696ce4c728d410b325dfe4099, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/817bbb8696494171be63b200e480be74, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/298b24d86eff42c285c52df630edc72c, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/e32b4b3110b04f12a891660d01b34d8c, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/a356c57983bf487eaa945a53843b801e] hfiles 2023-11-19T01:28:29,950 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (1/8): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757 for snapshot=snapshot_backup_system 2023-11-19T01:28:29,950 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (2/8): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63 for snapshot=snapshot_backup_system 2023-11-19T01:28:29,950 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (3/8): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2 for snapshot=snapshot_backup_system 2023-11-19T01:28:29,950 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (4/8): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/83b8d86696ce4c728d410b325dfe4099 for snapshot=snapshot_backup_system 2023-11-19T01:28:29,950 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (5/8): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/817bbb8696494171be63b200e480be74 for snapshot=snapshot_backup_system 2023-11-19T01:28:29,950 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (6/8): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/298b24d86eff42c285c52df630edc72c for snapshot=snapshot_backup_system 2023-11-19T01:28:29,950 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (7/8): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/e32b4b3110b04f12a891660d01b34d8c for snapshot=snapshot_backup_system 2023-11-19T01:28:29,950 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (8/8): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/a356c57983bf487eaa945a53843b801e for snapshot=snapshot_backup_system 2023-11-19T01:28:29,950 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/05f6b05b598842a0a5af3e603f2dfa9f, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/6e39103009374e778431dfd7316f5a4c, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/1b2851533774422da3cbc0adf48207b8, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/20bd2650ab284e0b9d4e6d4318baa9b6, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/4b8411bfe401452da4354337a869a577] hfiles 2023-11-19T01:28:29,950 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (1/9): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99 for snapshot=snapshot_backup_system 2023-11-19T01:28:29,950 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (2/9): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff for snapshot=snapshot_backup_system 2023-11-19T01:28:29,950 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (3/9): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460 for snapshot=snapshot_backup_system 2023-11-19T01:28:29,951 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (4/9): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1 for snapshot=snapshot_backup_system 2023-11-19T01:28:29,951 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (5/9): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/05f6b05b598842a0a5af3e603f2dfa9f for snapshot=snapshot_backup_system 2023-11-19T01:28:29,951 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (6/9): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/6e39103009374e778431dfd7316f5a4c for snapshot=snapshot_backup_system 2023-11-19T01:28:29,951 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (7/9): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/1b2851533774422da3cbc0adf48207b8 for snapshot=snapshot_backup_system 2023-11-19T01:28:29,951 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (8/9): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/20bd2650ab284e0b9d4e6d4318baa9b6 for snapshot=snapshot_backup_system 2023-11-19T01:28:29,951 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] snapshot.SnapshotManifest(264): Adding reference for file (9/9): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/4b8411bfe401452da4354337a869a577 for snapshot=snapshot_backup_system 2023-11-19T01:28:30,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=115 2023-11-19T01:28:30,364 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:28:30,365 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=116 2023-11-19T01:28:30,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=116 2023-11-19T01:28:30,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:28:30,366 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=116, ppid=115, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:28:30,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=116, resume processing ppid=115 2023-11-19T01:28:30,369 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:28:30,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=116, ppid=115, state=SUCCESS; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc in 1.0110 sec 2023-11-19T01:28:30,369 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:28:30,370 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:28:30,370 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:28:30,370 DEBUG [PEWorker-2 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:28:30,782 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:28:30,792 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:28:30,793 DEBUG [PEWorker-2 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:28:30,794 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=115, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:28:30,794 DEBUG [PEWorker-2 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 115 2023-11-19T01:28:30,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=115, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=115, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.8580 sec 2023-11-19T01:28:31,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=115 2023-11-19T01:28:31,045 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(115): Backup backup_1700357308898 started at 1700357311045. 2023-11-19T01:28:31,046 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(119): Backup session backup_1700357308898 has been started. 2023-11-19T01:28:31,054 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-19T01:28:31,055 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:28:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:28:31,055 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:28:31,055 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:28:31,056 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:28:31,056 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:28:31,057 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:28:31,059 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:31,059 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:28:31,059 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:28:31,059 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:28:31,059 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:28:31,060 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:28:31,060 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:28:31,060 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:28:31,060 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:28:31,060 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:28:31,060 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:28:31,060 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:28:31,061 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:28:31,061 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:28:31,061 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:28:31,064 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:28:31,064 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:31,064 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:28:31,064 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:28:31,064 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:28:31,064 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:28:31,064 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:28:31,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:28:31,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:31,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:28:31,066 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:28:31,066 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@41b29150[Count = 0] remaining members to acquire global barrier 2023-11-19T01:28:31,066 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:28:31,066 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:28:31,067 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:28:31,067 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:28:31,067 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:28:31,067 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:28:31,067 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:28:31,067 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:31,067 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357290490 highest: 1700357290490 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:31,068 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784:(num 1700357290490) roll requested 2023-11-19T01:28:31,079 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357311068, exclude list is [], retry=0 2023-11-19T01:28:31,083 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:28:31,085 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357290490 with entries=15, filesize=4.08 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357311068 2023-11-19T01:28:31,085 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:28:31,085 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357290490 is not closed yet, will try archiving it next time 2023-11-19T01:28:31,085 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta:.meta(num 1700357290516) roll requested 2023-11-19T01:28:31,085 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357275821 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/oldWALs/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357275821 2023-11-19T01:28:31,097 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357311085.meta, exclude list is [], retry=0 2023-11-19T01:28:31,101 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:28:31,102 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357290516.meta with entries=24, filesize=5.75 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357311085.meta 2023-11-19T01:28:31,103 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:28:31,103 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357290516.meta is not closed yet, will try archiving it next time 2023-11-19T01:28:31,108 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 41 2023-11-19T01:28:31,108 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357311068 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:31,116 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:28:31,116 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,44997,1700357176784' in zk 2023-11-19T01:28:31,119 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:28:31,119 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:28:31,119 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:28:31,120 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:31,120 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:28:31,120 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:28:31,121 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:28:31,121 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:28:31,121 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:28:31,122 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:31,122 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:28:31,122 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:28:31,122 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:31,123 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,44997,1700357176784': 2023-11-19T01:28:31,123 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:28:31,123 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:28:31,123 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:28:31,123 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:28:31,123 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:28:31,123 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:28:31,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:28:31,126 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:28:31,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:28:31,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:28:31,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:28:31,126 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:28:31,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:28:31,126 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:28:31,127 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:28:31,127 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:28:31,127 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:28:31,128 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:28:31,128 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:31,128 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:28:31,129 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:28:31,129 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:31,133 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:28:31,133 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:28:31,133 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:28:31,133 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:28:31,134 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:28:31,134 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:28:31,134 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:28:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-19T01:28:31,238 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357311237_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } 2023-11-19T01:28:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357311238 (current time:1700357311238). 2023-11-19T01:28:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:28:31,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357311237_ns2_test-17003571846791 VERSION not specified, setting to 2 2023-11-19T01:28:31,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ee9ff0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:31,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:31,247 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43488, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:31,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:31,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:28:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=117, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1700357311237_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } 2023-11-19T01:28:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357311237_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 }, snapshot procedure id = 117 2023-11-19T01:28:31,250 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1700357311237_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:28:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=117 2023-11-19T01:28:31,251 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1700357311237_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:28:31,254 WARN [PEWorker-3 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357311237_ns2_test-17003571846791 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:28:31,257 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1700357311237_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:28:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=117 2023-11-19T01:28:31,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=117 2023-11-19T01:28:31,662 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1700357311237_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:28:31,662 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE; SnapshotRegionProcedure 4ef412d65669b2fe1684b7fdf6ee1f7d}] 2023-11-19T01:28:31,664 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=118, ppid=117, state=RUNNABLE; SnapshotRegionProcedure 4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:28:31,815 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:31,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=118 2023-11-19T01:28:31,815 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. 2023-11-19T01:28:31,815 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.HRegion(2520): Flush status journal for 4ef412d65669b2fe1684b7fdf6ee1f7d: 2023-11-19T01:28:31,816 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. for snapshot_1700357311237_ns2_test-17003571846791 completed. 2023-11-19T01:28:31,816 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(240): Storing 'ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d.' region-info for snapshot=snapshot_1700357311237_ns2_test-17003571846791 2023-11-19T01:28:31,816 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:28:31,816 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_] hfiles 2023-11-19T01:28:31,816 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_ for snapshot=snapshot_1700357311237_ns2_test-17003571846791 2023-11-19T01:28:31,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=117 2023-11-19T01:28:32,221 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. 2023-11-19T01:28:32,221 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=118 2023-11-19T01:28:32,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=118 2023-11-19T01:28:32,222 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357311237_ns2_test-17003571846791 on region 4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:28:32,224 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=118, ppid=117, state=RUNNABLE; SnapshotRegionProcedure 4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:28:32,226 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=118, resume processing ppid=117 2023-11-19T01:28:32,226 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=118, ppid=117, state=SUCCESS; SnapshotRegionProcedure 4ef412d65669b2fe1684b7fdf6ee1f7d in 562 msec 2023-11-19T01:28:32,226 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1700357311237_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:28:32,226 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1700357311237_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:28:32,227 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1700357311237_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:28:32,227 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357311237_ns2_test-17003571846791 2023-11-19T01:28:32,228 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357311237_ns2_test-17003571846791 2023-11-19T01:28:32,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=117 2023-11-19T01:28:32,642 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1700357311237_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:28:32,647 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1700357311237_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:28:32,647 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357311237_ns2_test-17003571846791 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357311237_ns2_test-17003571846791 2023-11-19T01:28:32,648 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=117, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1700357311237_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:28:32,648 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357311237_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 }, snapshot procedure id = 117 2023-11-19T01:28:32,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=117, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=117, snapshot={ ss=snapshot_1700357311237_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } in 1.4000 sec 2023-11-19T01:28:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=117 2023-11-19T01:28:33,356 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357313356_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } 2023-11-19T01:28:33,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357313356 (current time:1700357313356). 2023-11-19T01:28:33,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:28:33,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357313356_ns3_test-17003571846792 VERSION not specified, setting to 2 2023-11-19T01:28:33,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47eecb8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:33,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:33,365 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43490, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:33,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:33,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:28:33,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=119, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1700357313356_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } 2023-11-19T01:28:33,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357313356_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 }, snapshot procedure id = 119 2023-11-19T01:28:33,369 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1700357313356_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:28:33,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=119 2023-11-19T01:28:33,369 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1700357313356_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:28:33,372 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357313356_ns3_test-17003571846792 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:28:33,374 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1700357313356_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:28:33,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=119 2023-11-19T01:28:33,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=119 2023-11-19T01:28:33,698 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:28:33,781 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1700357313356_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:28:33,781 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE; SnapshotRegionProcedure b78f59bec877d4d278f3912d6fb4631e}] 2023-11-19T01:28:33,783 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=120, ppid=119, state=RUNNABLE; SnapshotRegionProcedure b78f59bec877d4d278f3912d6fb4631e 2023-11-19T01:28:33,933 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:33,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=120 2023-11-19T01:28:33,934 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. 2023-11-19T01:28:33,934 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.HRegion(2520): Flush status journal for b78f59bec877d4d278f3912d6fb4631e: 2023-11-19T01:28:33,934 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. for snapshot_1700357313356_ns3_test-17003571846792 completed. 2023-11-19T01:28:33,934 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] snapshot.SnapshotManifest(240): Storing 'ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e.' region-info for snapshot=snapshot_1700357313356_ns3_test-17003571846792 2023-11-19T01:28:33,934 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:28:33,934 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-19T01:28:33,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=119 2023-11-19T01:28:34,340 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. 2023-11-19T01:28:34,340 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=120 2023-11-19T01:28:34,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=120 2023-11-19T01:28:34,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357313356_ns3_test-17003571846792 on region b78f59bec877d4d278f3912d6fb4631e 2023-11-19T01:28:34,342 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=120, ppid=119, state=RUNNABLE; SnapshotRegionProcedure b78f59bec877d4d278f3912d6fb4631e 2023-11-19T01:28:34,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=120, resume processing ppid=119 2023-11-19T01:28:34,344 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1700357313356_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:28:34,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=120, ppid=119, state=SUCCESS; SnapshotRegionProcedure b78f59bec877d4d278f3912d6fb4631e in 562 msec 2023-11-19T01:28:34,344 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1700357313356_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:28:34,344 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1700357313356_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:28:34,345 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357313356_ns3_test-17003571846792 2023-11-19T01:28:34,346 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357313356_ns3_test-17003571846792 2023-11-19T01:28:34,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=119 2023-11-19T01:28:34,758 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1700357313356_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:28:34,762 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1700357313356_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:28:34,764 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357313356_ns3_test-17003571846792 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357313356_ns3_test-17003571846792 2023-11-19T01:28:34,765 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=119, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1700357313356_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:28:34,765 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357313356_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 }, snapshot procedure id = 119 2023-11-19T01:28:34,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=119, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=119, snapshot={ ss=snapshot_1700357313356_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } in 1.3990 sec 2023-11-19T01:28:34,784 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2023-11-19T01:28:35,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=119 2023-11-19T01:28:35,475 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1700357308898 2023-11-19T01:28:35,475 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-19T01:28:35,475 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-11-19T01:28:35,475 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357308898_ns2:test-17003571846791 2023-11-19T01:28:35,475 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357311237_ns2_test-17003571846791 to hdfs://localhost:42739/backupUT/backup_1700357308898/ns2/test-17003571846791/ 2023-11-19T01:28:35,475 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:28:35,498 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:28:35,498 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,44889,1700357173572_998643828_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42739/backupUT/backup_1700357308898/ns2/test-17003571846791, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42739/backupUT/backup_1700357308898/ns2/test-17003571846791/.hbase-snapshot/.tmp/snapshot_1700357311237_ns2_test-17003571846791 2023-11-19T01:28:35,498 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af, snapshotDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357311237_ns2_test-17003571846791. 2023-11-19T01:28:35,506 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357311237_ns2_test-17003571846791 to hdfs://localhost:42739/backupUT/backup_1700357308898/ns2/test-17003571846791/.hbase-snapshot/.tmp/snapshot_1700357311237_ns2_test-17003571846791 2023-11-19T01:28:35,520 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:35,521 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:35,521 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:35,522 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:36,538 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-966562897908884901.jar 2023-11-19T01:28:36,539 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:36,539 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:36,539 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:36,540 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:36,540 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:36,540 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:36,541 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:36,541 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:36,541 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:36,542 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:28:36,543 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:28:36,543 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:28:36,543 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:28:36,544 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:28:36,544 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:28:36,545 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:28:36,545 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:28:36,545 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:28:36,546 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:28:36,546 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:28:36,546 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:28:36,547 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:36,547 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:36,548 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:36,548 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:36,549 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:36,549 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:36,550 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:36,592 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357311237_ns2_test-17003571846791' hfile list 2023-11-19T01:28:36,594 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-19T01:28:36,665 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357316630/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:28:36,665 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357316630/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:28:36,719 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-19T01:28:36,743 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_ output=hdfs://localhost:42739/backupUT/backup_1700357308898/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_ 2023-11-19T01:28:36,743 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 8.321M/sec 2023-11-19T01:28:37,697 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:28:37,698 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:28:37,704 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357311237_ns2_test-17003571846791 2023-11-19T01:28:37,704 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357311237_ns2_test-17003571846791 finished. 2023-11-19T01:28:37,704 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357308898_ns3:test-17003571846792 2023-11-19T01:28:37,704 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357313356_ns3_test-17003571846792 to hdfs://localhost:42739/backupUT/backup_1700357308898/ns3/test-17003571846792/ 2023-11-19T01:28:37,704 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:28:37,727 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:28:37,727 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,44889,1700357173572_1411587021_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42739/backupUT/backup_1700357308898/ns3/test-17003571846792, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42739/backupUT/backup_1700357308898/ns3/test-17003571846792/.hbase-snapshot/.tmp/snapshot_1700357313356_ns3_test-17003571846792 2023-11-19T01:28:37,727 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af, snapshotDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357313356_ns3_test-17003571846792. 2023-11-19T01:28:37,731 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357313356_ns3_test-17003571846792 to hdfs://localhost:42739/backupUT/backup_1700357308898/ns3/test-17003571846792/.hbase-snapshot/.tmp/snapshot_1700357313356_ns3_test-17003571846792 2023-11-19T01:28:38,145 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:38,146 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:38,147 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:38,147 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:39,174 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-7147328245214612089.jar 2023-11-19T01:28:39,174 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:39,175 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:39,175 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:39,176 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:39,176 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:39,176 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:39,177 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:39,177 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:39,177 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:39,178 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:28:39,179 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:28:39,179 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:28:39,179 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:28:39,180 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:28:39,180 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:28:39,180 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:28:39,181 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:28:39,181 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:28:39,181 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:28:39,182 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:28:39,182 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:28:39,183 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:39,183 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:39,184 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:39,184 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:39,184 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:39,185 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:39,186 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:39,228 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357313356_ns3_test-17003571846792' hfile list 2023-11-19T01:28:39,316 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357319272/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:28:39,317 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357319272/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:28:40,289 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:28:40,345 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:28:40,346 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:28:40,350 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357313356_ns3_test-17003571846792 2023-11-19T01:28:40,350 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357313356_ns3_test-17003571846792 finished. 2023-11-19T01:28:40,350 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1034): ns2:test-17003571846791 2023-11-19T01:28:40,350 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1034): ns3:test-17003571846792 2023-11-19T01:28:40,356 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357308898 2023-11-19T01:28:40,356 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:28:40,761 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42739/backupUT/backup_1700357308898/.backup.manifest 2023-11-19T01:28:40,761 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357308898 2023-11-19T01:28:40,761 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:28:41,167 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42739/backupUT/backup_1700357308898/.backup.manifest 2023-11-19T01:28:41,167 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(358): Backup backup_1700357308898 finished: type=FULL,tablelist=ns2:test-17003571846791;ns3:test-17003571846792,targetRootDir=hdfs://localhost:42739/backupUT,startts=1700357311045,completets=1700357320356,bytescopied=0 2023-11-19T01:28:41,167 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-19T01:28:41,167 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357311237_ns2_test-17003571846791 2023-11-19T01:28:41,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357311237_ns2_test-17003571846791" 2023-11-19T01:28:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357311237_ns2_test-17003571846791 2023-11-19T01:28:41,171 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357311237_ns2_test-17003571846791 for backup backup_1700357308898 succeeded. 2023-11-19T01:28:41,171 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357313356_ns3_test-17003571846792 2023-11-19T01:28:41,171 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357313356_ns3_test-17003571846792" 2023-11-19T01:28:41,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357313356_ns3_test-17003571846792 2023-11-19T01:28:41,174 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357313356_ns3_test-17003571846792 for backup backup_1700357308898 succeeded. 2023-11-19T01:28:41,175 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:28:41,178 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:28:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:28:41,180 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:28:41,182 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:28:41,187 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(376): Backup backup_1700357308898 completed. 2023-11-19T01:28:41,188 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:41,197 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] backup.RestoreDriver(101): Found -overwrite option in restore command, will overwrite to existing table if any in the restore target 2023-11-19T01:28:41,202 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@114a8e3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:41,203 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:41,204 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:57128, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:41,208 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357308898/.backup.manifest 2023-11-19T01:28:41,210 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357308898/.backup.manifest 2023-11-19T01:28:41,210 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357308898/.backup.manifest 2023-11-19T01:28:41,211 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357308898/.backup.manifest 2023-11-19T01:28:41,212 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357308898/.backup.manifest 2023-11-19T01:28:41,213 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357308898/.backup.manifest 2023-11-19T01:28:41,213 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-17003571846791' to 'ns2:test-17003571846791' from full backup image hdfs://localhost:42739/backupUT/backup_1700357308898/ns2/test-17003571846791 2023-11-19T01:28:41,220 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(476): Truncating exising target table 'ns2:test-17003571846791', preserving region splits 2023-11-19T01:28:41,221 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:28:41,222 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:33066, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:28:41,223 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable ns2:test-17003571846791 2023-11-19T01:28:41,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=121, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:test-17003571846791 2023-11-19T01:28:41,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=121 2023-11-19T01:28:41,226 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-17003571846791","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357321226"}]},"ts":"1700357321226"} 2023-11-19T01:28:41,227 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-17003571846791, state=DISABLING in hbase:meta 2023-11-19T01:28:41,229 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(293): Set ns2:test-17003571846791 to state=DISABLING 2023-11-19T01:28:41,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-17003571846791, region=4ef412d65669b2fe1684b7fdf6ee1f7d, UNASSIGN}] 2023-11-19T01:28:41,232 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=122, ppid=121, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:test-17003571846791, region=4ef412d65669b2fe1684b7fdf6ee1f7d, UNASSIGN 2023-11-19T01:28:41,232 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=122 updating hbase:meta row=4ef412d65669b2fe1684b7fdf6ee1f7d, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:41,233 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE; CloseRegionProcedure 4ef412d65669b2fe1684b7fdf6ee1f7d, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:28:41,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=121 2023-11-19T01:28:41,385 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:41,386 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] handler.UnassignRegionHandler(119): Close 4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:28:41,386 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1668): Closing 4ef412d65669b2fe1684b7fdf6ee1f7d, disabling compactions & flushes 2023-11-19T01:28:41,386 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1690): Closing region ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. 2023-11-19T01:28:41,386 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. 2023-11-19T01:28:41,386 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1778): Acquired close lock on ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. after waiting 0 ms 2023-11-19T01:28:41,386 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1788): Updates disabled for region ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. 2023-11-19T01:28:41,392 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2023-11-19T01:28:41,393 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:28:41,394 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1909): Closed ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d. 2023-11-19T01:28:41,394 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] regionserver.HRegion(1622): Region close journal for 4ef412d65669b2fe1684b7fdf6ee1f7d: 2023-11-19T01:28:41,395 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=123}] handler.UnassignRegionHandler(163): Closed 4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:28:41,395 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=122 updating hbase:meta row=4ef412d65669b2fe1684b7fdf6ee1f7d, regionState=CLOSED 2023-11-19T01:28:41,397 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=123, resume processing ppid=122 2023-11-19T01:28:41,398 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=123, ppid=122, state=SUCCESS; CloseRegionProcedure 4ef412d65669b2fe1684b7fdf6ee1f7d, server=jenkins-hbase6.apache.org,44997,1700357176784 in 163 msec 2023-11-19T01:28:41,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=122, resume processing ppid=121 2023-11-19T01:28:41,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=122, ppid=121, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-17003571846791, region=4ef412d65669b2fe1684b7fdf6ee1f7d, UNASSIGN in 168 msec 2023-11-19T01:28:41,399 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-17003571846791","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357321399"}]},"ts":"1700357321399"} 2023-11-19T01:28:41,400 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-17003571846791, state=DISABLED in hbase:meta 2023-11-19T01:28:41,402 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(305): Set ns2:test-17003571846791 to state=DISABLED 2023-11-19T01:28:41,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=121, state=SUCCESS; DisableTableProcedure table=ns2:test-17003571846791 in 180 msec 2023-11-19T01:28:41,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=121 2023-11-19T01:28:41,528 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$6(2481): Client=jenkins//172.31.2.20 truncate ns2:test-17003571846791 2023-11-19T01:28:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=124, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns2:test-17003571846791 preserveSplits=true) 2023-11-19T01:28:41,531 DEBUG [PEWorker-2 {}] procedure.TruncateTableProcedure(87): waiting for 'ns2:test-17003571846791' regions in transition 2023-11-19T01:28:41,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=124 2023-11-19T01:28:41,533 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:28:41,535 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f, FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/recovered.edits] 2023-11-19T01:28:41,539 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_ to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/archive/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_ 2023-11-19T01:28:41,542 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/recovered.edits/9.seqid to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/archive/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/recovered.edits/9.seqid 2023-11-19T01:28:41,543 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:28:41,543 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived ns2:test-17003571846791 regions 2023-11-19T01:28:41,546 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:test-17003571846791 from hbase:meta 2023-11-19T01:28:41,548 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:test-17003571846791' descriptor. 2023-11-19T01:28:41,548 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:test-17003571846791' from region states. 2023-11-19T01:28:41,548 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357321548"}]},"ts":"9223372036854775807"} 2023-11-19T01:28:41,550 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:28:41,550 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 4ef412d65669b2fe1684b7fdf6ee1f7d, NAME => 'ns2:test-17003571846791,,1700357208704.4ef412d65669b2fe1684b7fdf6ee1f7d.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:28:41,550 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:test-17003571846791' as deleted. 2023-11-19T01:28:41,550 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:test-17003571846791","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1700357321550"}]},"ts":"9223372036854775807"} 2023-11-19T01:28:41,551 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:test-17003571846791 state from META 2023-11-19T01:28:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=124 2023-11-19T01:28:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=124 2023-11-19T01:28:41,962 INFO [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => a169e45a4f1d98e65b6a1dada440fc40, NAME => 'ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:test-17003571846791', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:28:42,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=124 2023-11-19T01:28:42,373 DEBUG [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:28:42,373 DEBUG [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1668): Closing a169e45a4f1d98e65b6a1dada440fc40, disabling compactions & flushes 2023-11-19T01:28:42,373 INFO [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40. 2023-11-19T01:28:42,373 DEBUG [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40. 2023-11-19T01:28:42,373 DEBUG [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40. after waiting 0 ms 2023-11-19T01:28:42,373 DEBUG [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40. 2023-11-19T01:28:42,373 INFO [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1909): Closed ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40. 2023-11-19T01:28:42,373 DEBUG [RegionOpenAndInit-ns2:test-17003571846791-pool-0 {}] regionserver.HRegion(1622): Region close journal for a169e45a4f1d98e65b6a1dada440fc40: 2023-11-19T01:28:42,374 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1700357322374"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357322374"}]},"ts":"1700357322374"} 2023-11-19T01:28:42,375 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:28:42,376 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-17003571846791","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357322375"}]},"ts":"1700357322375"} 2023-11-19T01:28:42,376 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-17003571846791, state=ENABLING in hbase:meta 2023-11-19T01:28:42,381 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-17003571846791, region=a169e45a4f1d98e65b6a1dada440fc40, ASSIGN}] 2023-11-19T01:28:42,382 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:test-17003571846791, region=a169e45a4f1d98e65b6a1dada440fc40, ASSIGN 2023-11-19T01:28:42,383 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(262): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:test-17003571846791, region=a169e45a4f1d98e65b6a1dada440fc40, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,44997,1700357176784; forceNewPlan=false, retain=false 2023-11-19T01:28:42,534 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=125 updating hbase:meta row=a169e45a4f1d98e65b6a1dada440fc40, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:42,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE; OpenRegionProcedure a169e45a4f1d98e65b6a1dada440fc40, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:28:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=124 2023-11-19T01:28:42,687 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:42,690 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(130): Open ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40. 2023-11-19T01:28:42,690 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7254): Opening region: {ENCODED => a169e45a4f1d98e65b6a1dada440fc40, NAME => 'ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:28:42,691 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:28:42,691 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17003571846791 a169e45a4f1d98e65b6a1dada440fc40 2023-11-19T01:28:42,691 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(888): Instantiated ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:28:42,691 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7296): checking encryption for a169e45a4f1d98e65b6a1dada440fc40 2023-11-19T01:28:42,691 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7299): checking classloading for a169e45a4f1d98e65b6a1dada440fc40 2023-11-19T01:28:42,693 INFO [StoreOpener-a169e45a4f1d98e65b6a1dada440fc40-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region a169e45a4f1d98e65b6a1dada440fc40 2023-11-19T01:28:42,694 INFO [StoreOpener-a169e45a4f1d98e65b6a1dada440fc40-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a169e45a4f1d98e65b6a1dada440fc40 columnFamilyName f 2023-11-19T01:28:42,694 DEBUG [StoreOpener-a169e45a4f1d98e65b6a1dada440fc40-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:28:42,695 INFO [StoreOpener-a169e45a4f1d98e65b6a1dada440fc40-1 {}] regionserver.HStore(324): Store=a169e45a4f1d98e65b6a1dada440fc40/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:28:42,695 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40 2023-11-19T01:28:42,696 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40 2023-11-19T01:28:42,698 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1076): writing seq id for a169e45a4f1d98e65b6a1dada440fc40 2023-11-19T01:28:42,699 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:28:42,700 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): Opened a169e45a4f1d98e65b6a1dada440fc40; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11243824960, jitterRate=0.047162801027297974}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:28:42,700 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(992): Region open journal for a169e45a4f1d98e65b6a1dada440fc40: 2023-11-19T01:28:42,701 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40., pid=126, masterSystemTime=1700357322687 2023-11-19T01:28:42,702 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40. 2023-11-19T01:28:42,702 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(158): Opened ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40. 2023-11-19T01:28:42,703 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=125 updating hbase:meta row=a169e45a4f1d98e65b6a1dada440fc40, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:42,703 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44889 {}] assignment.AssignmentManager(1430): Unable to acquire lock for regionNode state=OPEN, location=jenkins-hbase6.apache.org,44997,1700357176784, table=ns2:test-17003571846791, region=a169e45a4f1d98e65b6a1dada440fc40. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2023-11-19T01:28:42,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=126, resume processing ppid=125 2023-11-19T01:28:42,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=126, ppid=125, state=SUCCESS; OpenRegionProcedure a169e45a4f1d98e65b6a1dada440fc40, server=jenkins-hbase6.apache.org,44997,1700357176784 in 169 msec 2023-11-19T01:28:42,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=125, resume processing ppid=124 2023-11-19T01:28:42,706 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=125, ppid=124, state=SUCCESS; TransitRegionStateProcedure table=ns2:test-17003571846791, region=a169e45a4f1d98e65b6a1dada440fc40, ASSIGN in 324 msec 2023-11-19T01:28:42,706 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:test-17003571846791","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357322706"}]},"ts":"1700357322706"} 2023-11-19T01:28:42,707 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:test-17003571846791, state=ENABLED in hbase:meta 2023-11-19T01:28:42,709 DEBUG [PEWorker-3 {}] procedure.TruncateTableProcedure(145): truncate 'ns2:test-17003571846791' completed 2023-11-19T01:28:42,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=124, state=SUCCESS; TruncateTableProcedure (table=ns2:test-17003571846791 preserveSplits=true) in 1.1810 sec 2023-11-19T01:28:43,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=124 2023-11-19T01:28:43,641 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:42739/backupUT/backup_1700357308898/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d from hbase tables ns2:test-17003571846791 to tables ns2:test-17003571846791 2023-11-19T01:28:43,641 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-17003571846791 into ns2:test-17003571846791 2023-11-19T01:28:43,643 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-test-17003571846791-1700357323641 from hdfs://localhost:42739/backupUT/backup_1700357308898/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d 2023-11-19T01:28:43,650 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f10ba14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:43,651 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:28:43,652 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:33078, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:28:43,653 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-19T01:28:43,653 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:test-17003571846791 2023-11-19T01:28:43,656 DEBUG [hconnection-0x10d246f9-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:43,657 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:57142, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:43,658 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:test-17003571846791: 2023-11-19T01:28:43,658 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-11-19T01:28:43,658 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/partitions_9ea4b2f8-fddb-4e23-a23a-e3bbd821af2e 2023-11-19T01:28:44,065 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:44,065 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:44,066 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:44,066 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:45,096 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-6264050509523428988.jar 2023-11-19T01:28:45,097 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:45,097 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:45,097 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:45,098 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:45,098 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:45,098 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:45,099 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:45,099 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:45,100 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:45,101 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:28:45,102 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:28:45,102 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:28:45,102 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:28:45,103 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:28:45,104 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:28:45,104 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:28:45,104 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:28:45,105 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:28:45,105 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:28:45,106 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:28:45,106 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:28:45,107 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:45,107 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:45,107 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:45,108 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:45,108 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:45,108 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:45,109 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:45,109 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:test-17003571846791 2023-11-19T01:28:45,109 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:45,109 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-11-19T01:28:45,109 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-11-19T01:28:45,110 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:28:45,123 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-11-19T01:28:45,211 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357325171/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:28:45,211 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357325171/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:28:45,261 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:42739/backupUT/backup_1700357308898/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/4ef412d65669b2fe1684b7fdf6ee1f7d/f/6157b1620f8c4c3281188261bf9ca9af_SeqId_4_ 2023-11-19T01:28:45,264 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-11-19T01:28:45,285 DEBUG [pool-441-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd297d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:45,287 DEBUG [hconnection-0x710ac409-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:45,289 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:57156, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:45,293 DEBUG [pool-441-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:45,293 DEBUG [pool-441-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-11-19T01:28:45,293 DEBUG [pool-441-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase6.apache.org 2023-11-19T01:28:45,304 DEBUG [pool-441-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-test-17003571846791-1700357323641/_temporary/0/_temporary/attempt_local750857031_0019_r_000000_0/f/bceac88375e84549b8b3580d2d6541b7 is 37, key is row10/f:q1/1700357188531/Put/seqid=0 2023-11-19T01:28:45,305 WARN [Thread-1917 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase6.apache.org:44997] Specified favored nodes: [jenkins-hbase6.apache.org:44997] 2023-11-19T01:28:46,258 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-test-17003571846791-1700357323641 2023-11-19T01:28:46,264 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16d278dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:46,265 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:46,267 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:57158, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:46,272 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-test-17003571846791-1700357323641/_SUCCESS 2023-11-19T01:28:46,276 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:28:46,278 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:33084, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:28:46,288 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-test-17003571846791-1700357323641/f/bceac88375e84549b8b3580d2d6541b7 first=Optional[row0] last=Optional[row98] 2023-11-19T01:28:46,290 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40., hostname=jenkins-hbase6.apache.org,44997,1700357176784, seqNum=2 for row with hfile group [{f,hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-test-17003571846791-1700357323641/f/bceac88375e84549b8b3580d2d6541b7}] 2023-11-19T01:28:46,308 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-test-17003571846791-1700357323641/f/bceac88375e84549b8b3580d2d6541b7 for inclusion in a169e45a4f1d98e65b6a1dada440fc40/f 2023-11-19T01:28:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-11-19T01:28:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(625): Region bounds: first= last= 2023-11-19T01:28:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegion(2520): Flush status journal for a169e45a4f1d98e65b6a1dada440fc40: 2023-11-19T01:28:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-test-17003571846791-1700357323641/f/bceac88375e84549b8b3580d2d6541b7 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__ns2_test-17003571846791__99arqh5g2kb46qrmafv2ea77qk41konjqns2bnean8k6ukus0hcn4rbk0hso62c5/f/bceac88375e84549b8b3580d2d6541b7 2023-11-19T01:28:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__ns2_test-17003571846791__99arqh5g2kb46qrmafv2ea77qk41konjqns2bnean8k6ukus0hcn4rbk0hso62c5/f/bceac88375e84549b8b3580d2d6541b7 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40/f/9301f8758b1e47ab85817026d7156af0_SeqId_4_ 2023-11-19T01:28:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52a12bb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:28:46,330 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:33100, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:28:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:46,335 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:57174, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(428): write bulk load descriptor to backup ns2:test-17003571846791 with 1 entries 2023-11-19T01:28:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(1602): writing raw bulk path hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40/f/9301f8758b1e47ab85817026d7156af0_SeqId_4_ for ns2:test-17003571846791 a169e45a4f1d98e65b6a1dada440fc40 2023-11-19T01:28:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(435): written 1 rows for bulk load of ns2:test-17003571846791 2023-11-19T01:28:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:46,350 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__ns2_test-17003571846791__99arqh5g2kb46qrmafv2ea77qk41konjqns2bnean8k6ukus0hcn4rbk0hso62c5/f/bceac88375e84549b8b3580d2d6541b7 into a169e45a4f1d98e65b6a1dada440fc40/f as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40/f/9301f8758b1e47ab85817026d7156af0_SeqId_4_ - updating store file list. 2023-11-19T01:28:46,356 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40/f/9301f8758b1e47ab85817026d7156af0_SeqId_4_ into a169e45a4f1d98e65b6a1dada440fc40/f 2023-11-19T01:28:46,356 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__ns2_test-17003571846791__99arqh5g2kb46qrmafv2ea77qk41konjqns2bnean8k6ukus0hcn4rbk0hso62c5/f/bceac88375e84549b8b3580d2d6541b7 into a169e45a4f1d98e65b6a1dada440fc40/f (new location: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40/f/9301f8758b1e47ab85817026d7156af0_SeqId_4_) 2023-11-19T01:28:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__ns2_test-17003571846791__99arqh5g2kb46qrmafv2ea77qk41konjqns2bnean8k6ukus0hcn4rbk0hso62c5/f/bceac88375e84549b8b3580d2d6541b7 2023-11-19T01:28:46,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28bbcdc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:28:46,365 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:33110, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-19T01:28:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:46,371 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:57184, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-11-19T01:28:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(408): write bulk load descriptor to backup ns2:test-17003571846791 with 1 entries 2023-11-19T01:28:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(1523): writing done bulk path hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40/f/9301f8758b1e47ab85817026d7156af0_SeqId_4_ for ns2:test-17003571846791 a169e45a4f1d98e65b6a1dada440fc40 2023-11-19T01:28:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] impl.BackupSystemTable(414): written 1 rows for bulk load of ns2:test-17003571846791 2023-11-19T01:28:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:46,396 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:46,396 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-11-19T01:28:46,397 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-19T01:28:46,397 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(233): Backup: backup_1700357308898 hdfs://localhost:42739/backupUT/backup_1700357308898/ns2/test-17003571846791/ 2023-11-19T01:28:46,399 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357308898/.backup.manifest 2023-11-19T01:28:46,401 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357308898/.backup.manifest 2023-11-19T01:28:46,401 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-17003571846792' to 'ns3:test-17003571846792' from full backup image hdfs://localhost:42739/backupUT/backup_1700357308898/ns3/test-17003571846792 2023-11-19T01:28:46,406 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:42739/backupUT/backup_1700357308898/ns3/test-17003571846792/archive/data/ns3/test-17003571846792 does not exists 2023-11-19T01:28:46,406 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-17003571846792, will only create table 2023-11-19T01:28:46,407 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(476): Truncating exising target table 'ns3:test-17003571846792', preserving region splits 2023-11-19T01:28:46,407 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable ns3:test-17003571846792 2023-11-19T01:28:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=127, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:test-17003571846792 2023-11-19T01:28:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=127 2023-11-19T01:28:46,411 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-17003571846792","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357326411"}]},"ts":"1700357326411"} 2023-11-19T01:28:46,413 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-17003571846792, state=DISABLING in hbase:meta 2023-11-19T01:28:46,414 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(293): Set ns3:test-17003571846792 to state=DISABLING 2023-11-19T01:28:46,415 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-17003571846792, region=b78f59bec877d4d278f3912d6fb4631e, UNASSIGN}] 2023-11-19T01:28:46,416 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=128, ppid=127, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:test-17003571846792, region=b78f59bec877d4d278f3912d6fb4631e, UNASSIGN 2023-11-19T01:28:46,417 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=128 updating hbase:meta row=b78f59bec877d4d278f3912d6fb4631e, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:46,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE; CloseRegionProcedure b78f59bec877d4d278f3912d6fb4631e, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:28:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=127 2023-11-19T01:28:46,569 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:46,569 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] handler.UnassignRegionHandler(119): Close b78f59bec877d4d278f3912d6fb4631e 2023-11-19T01:28:46,570 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1668): Closing b78f59bec877d4d278f3912d6fb4631e, disabling compactions & flushes 2023-11-19T01:28:46,570 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1690): Closing region ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. 2023-11-19T01:28:46,570 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. 2023-11-19T01:28:46,570 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1778): Acquired close lock on ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. after waiting 0 ms 2023-11-19T01:28:46,570 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1788): Updates disabled for region ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. 2023-11-19T01:28:46,573 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/b78f59bec877d4d278f3912d6fb4631e/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=1 2023-11-19T01:28:46,574 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:28:46,575 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1909): Closed ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e. 2023-11-19T01:28:46,575 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] regionserver.HRegion(1622): Region close journal for b78f59bec877d4d278f3912d6fb4631e: 2023-11-19T01:28:46,577 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=129}] handler.UnassignRegionHandler(163): Closed b78f59bec877d4d278f3912d6fb4631e 2023-11-19T01:28:46,577 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=128 updating hbase:meta row=b78f59bec877d4d278f3912d6fb4631e, regionState=CLOSED 2023-11-19T01:28:46,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=129, resume processing ppid=128 2023-11-19T01:28:46,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=129, ppid=128, state=SUCCESS; CloseRegionProcedure b78f59bec877d4d278f3912d6fb4631e, server=jenkins-hbase6.apache.org,44997,1700357176784 in 160 msec 2023-11-19T01:28:46,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=128, resume processing ppid=127 2023-11-19T01:28:46,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=128, ppid=127, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-17003571846792, region=b78f59bec877d4d278f3912d6fb4631e, UNASSIGN in 164 msec 2023-11-19T01:28:46,581 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-17003571846792","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357326581"}]},"ts":"1700357326581"} 2023-11-19T01:28:46,582 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-17003571846792, state=DISABLED in hbase:meta 2023-11-19T01:28:46,584 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(305): Set ns3:test-17003571846792 to state=DISABLED 2023-11-19T01:28:46,586 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=127, state=SUCCESS; DisableTableProcedure table=ns3:test-17003571846792 in 178 msec 2023-11-19T01:28:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=127 2023-11-19T01:28:46,715 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$6(2481): Client=jenkins//172.31.2.20 truncate ns3:test-17003571846792 2023-11-19T01:28:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=130, state=RUNNABLE:TRUNCATE_TABLE_PRE_OPERATION; TruncateTableProcedure (table=ns3:test-17003571846792 preserveSplits=true) 2023-11-19T01:28:46,718 DEBUG [PEWorker-4 {}] procedure.TruncateTableProcedure(87): waiting for 'ns3:test-17003571846792' regions in transition 2023-11-19T01:28:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=130 2023-11-19T01:28:46,720 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/b78f59bec877d4d278f3912d6fb4631e 2023-11-19T01:28:46,722 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/b78f59bec877d4d278f3912d6fb4631e/f, FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/b78f59bec877d4d278f3912d6fb4631e/recovered.edits] 2023-11-19T01:28:46,726 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/b78f59bec877d4d278f3912d6fb4631e/recovered.edits/7.seqid to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/archive/data/ns3/test-17003571846792/b78f59bec877d4d278f3912d6fb4631e/recovered.edits/7.seqid 2023-11-19T01:28:46,727 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/b78f59bec877d4d278f3912d6fb4631e 2023-11-19T01:28:46,727 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(293): Archived ns3:test-17003571846792 regions 2023-11-19T01:28:46,731 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:test-17003571846792 from hbase:meta 2023-11-19T01:28:46,733 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:test-17003571846792' descriptor. 2023-11-19T01:28:46,733 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:test-17003571846792' from region states. 2023-11-19T01:28:46,733 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357326733"}]},"ts":"9223372036854775807"} 2023-11-19T01:28:46,735 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:28:46,735 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => b78f59bec877d4d278f3912d6fb4631e, NAME => 'ns3:test-17003571846792,,1700357214141.b78f59bec877d4d278f3912d6fb4631e.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:28:46,735 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:test-17003571846792' as deleted. 2023-11-19T01:28:46,735 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:test-17003571846792","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1700357326735"}]},"ts":"9223372036854775807"} 2023-11-19T01:28:46,737 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:test-17003571846792 state from META 2023-11-19T01:28:46,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=130 2023-11-19T01:28:46,896 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:28:47,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=130 2023-11-19T01:28:47,147 INFO [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 09936870cc2abdecec74e26057d016f7, NAME => 'ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:test-17003571846792', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:28:47,152 DEBUG [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:28:47,152 DEBUG [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1668): Closing 09936870cc2abdecec74e26057d016f7, disabling compactions & flushes 2023-11-19T01:28:47,152 INFO [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7. 2023-11-19T01:28:47,152 DEBUG [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7. 2023-11-19T01:28:47,153 DEBUG [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7. after waiting 0 ms 2023-11-19T01:28:47,153 DEBUG [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7. 2023-11-19T01:28:47,153 INFO [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1909): Closed ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7. 2023-11-19T01:28:47,153 DEBUG [RegionOpenAndInit-ns3:test-17003571846792-pool-0 {}] regionserver.HRegion(1622): Region close journal for 09936870cc2abdecec74e26057d016f7: 2023-11-19T01:28:47,154 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7.","families":{"info":[{"qualifier":"regioninfo","vlen":49,"tag":[],"timestamp":"1700357327154"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357327154"}]},"ts":"1700357327154"} 2023-11-19T01:28:47,155 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:28:47,156 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-17003571846792","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357327156"}]},"ts":"1700357327156"} 2023-11-19T01:28:47,157 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-17003571846792, state=ENABLING in hbase:meta 2023-11-19T01:28:47,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-17003571846792, region=09936870cc2abdecec74e26057d016f7, ASSIGN}] 2023-11-19T01:28:47,164 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:test-17003571846792, region=09936870cc2abdecec74e26057d016f7, ASSIGN 2023-11-19T01:28:47,165 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(262): Starting pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:test-17003571846792, region=09936870cc2abdecec74e26057d016f7, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,44997,1700357176784; forceNewPlan=false, retain=false 2023-11-19T01:28:47,316 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=131 updating hbase:meta row=09936870cc2abdecec74e26057d016f7, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:47,318 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE; OpenRegionProcedure 09936870cc2abdecec74e26057d016f7, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:28:47,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=130 2023-11-19T01:28:47,470 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:47,474 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(130): Open ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7. 2023-11-19T01:28:47,474 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7254): Opening region: {ENCODED => 09936870cc2abdecec74e26057d016f7, NAME => 'ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:28:47,474 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:28:47,475 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test-17003571846792 09936870cc2abdecec74e26057d016f7 2023-11-19T01:28:47,475 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(888): Instantiated ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:28:47,475 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7296): checking encryption for 09936870cc2abdecec74e26057d016f7 2023-11-19T01:28:47,475 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7299): checking classloading for 09936870cc2abdecec74e26057d016f7 2023-11-19T01:28:47,476 INFO [StoreOpener-09936870cc2abdecec74e26057d016f7-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 09936870cc2abdecec74e26057d016f7 2023-11-19T01:28:47,478 INFO [StoreOpener-09936870cc2abdecec74e26057d016f7-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09936870cc2abdecec74e26057d016f7 columnFamilyName f 2023-11-19T01:28:47,478 DEBUG [StoreOpener-09936870cc2abdecec74e26057d016f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:28:47,478 INFO [StoreOpener-09936870cc2abdecec74e26057d016f7-1 {}] regionserver.HStore(324): Store=09936870cc2abdecec74e26057d016f7/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:28:47,479 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/09936870cc2abdecec74e26057d016f7 2023-11-19T01:28:47,479 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/09936870cc2abdecec74e26057d016f7 2023-11-19T01:28:47,482 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1076): writing seq id for 09936870cc2abdecec74e26057d016f7 2023-11-19T01:28:47,487 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/09936870cc2abdecec74e26057d016f7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:28:47,488 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1093): Opened 09936870cc2abdecec74e26057d016f7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10168780640, jitterRate=-0.05295850336551666}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:28:47,488 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(992): Region open journal for 09936870cc2abdecec74e26057d016f7: 2023-11-19T01:28:47,489 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7., pid=132, masterSystemTime=1700357327470 2023-11-19T01:28:47,491 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7. 2023-11-19T01:28:47,491 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(158): Opened ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7. 2023-11-19T01:28:47,491 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=131 updating hbase:meta row=09936870cc2abdecec74e26057d016f7, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:47,494 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=132, resume processing ppid=131 2023-11-19T01:28:47,494 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=132, ppid=131, state=SUCCESS; OpenRegionProcedure 09936870cc2abdecec74e26057d016f7, server=jenkins-hbase6.apache.org,44997,1700357176784 in 175 msec 2023-11-19T01:28:47,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=131, resume processing ppid=130 2023-11-19T01:28:47,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=131, ppid=130, state=SUCCESS; TransitRegionStateProcedure table=ns3:test-17003571846792, region=09936870cc2abdecec74e26057d016f7, ASSIGN in 332 msec 2023-11-19T01:28:47,496 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:test-17003571846792","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357327496"}]},"ts":"1700357327496"} 2023-11-19T01:28:47,497 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:test-17003571846792, state=ENABLED in hbase:meta 2023-11-19T01:28:47,499 DEBUG [PEWorker-1 {}] procedure.TruncateTableProcedure(145): truncate 'ns3:test-17003571846792' completed 2023-11-19T01:28:47,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=130, state=SUCCESS; TruncateTableProcedure (table=ns3:test-17003571846792 preserveSplits=true) in 784 msec 2023-11-19T01:28:47,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=130 2023-11-19T01:28:47,826 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-19T01:28:47,826 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(233): Backup: backup_1700357308898 hdfs://localhost:42739/backupUT/backup_1700357308898/ns3/test-17003571846792/ 2023-11-19T01:28:47,826 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-11-19T01:28:47,826 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:47,846 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleOverwriteCommand Thread=607 (was 602) - Thread LEAK? -, OpenFileDescriptor=712 (was 704) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=104 (was 114), ProcessCount=166 (was 167), AvailableMemoryMB=4424 (was 3529) - AvailableMemoryMB LEAK? - 2023-11-19T01:28:47,846 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=607 is superior to 500 2023-11-19T01:28:47,861 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleCommand Thread=607, OpenFileDescriptor=712, MaxFileDescriptor=60000, SystemLoadAverage=104, ProcessCount=166, AvailableMemoryMB=4423 2023-11-19T01:28:47,861 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=607 is superior to 500 2023-11-19T01:28:47,862 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestFullRestore(144): create full backup image on multiple tables: command-line 2023-11-19T01:28:47,885 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@488b322, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:47,889 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:47,892 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:57192, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:47,894 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:28:47,896 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:33114, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:28:47,910 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:28:47,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:28:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357327919 (current time:1700357327919). 2023-11-19T01:28:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:28:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:28:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30520c01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:47,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:47,930 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:57204, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:47,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:28:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=133, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:28:47,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 133 2023-11-19T01:28:47,936 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:28:47,937 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:28:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=133 2023-11-19T01:28:47,943 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:28:47,946 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:28:47,968 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:28:47,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc}] 2023-11-19T01:28:47,970 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=134, ppid=133, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:28:48,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=133 2023-11-19T01:28:48,121 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:48,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=134 2023-11-19T01:28:48,121 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:28:48,121 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegion(2819): Flushing 322466352cac3ade468ab2ce8e706acc 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-11-19T01:28:48,125 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/10d7714ea1534d16b9a707aa59fac0a0 is 171, key is trslm:hdfs://localhost:42739/backupUT\x00ns2:test-17003571846791/meta:log-roll-map/1700357320352/Put/seqid=0 2023-11-19T01:28:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=133 2023-11-19T01:28:48,529 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/10d7714ea1534d16b9a707aa59fac0a0 2023-11-19T01:28:48,536 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/5396f5846ade4d52893b3744b3e7409e is 524, key is session:backup_1700357308898/session:context/1700357321181/Put/seqid=0 2023-11-19T01:28:48,540 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/5396f5846ade4d52893b3744b3e7409e 2023-11-19T01:28:48,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=133 2023-11-19T01:28:48,546 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/10d7714ea1534d16b9a707aa59fac0a0 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/10d7714ea1534d16b9a707aa59fac0a0 2023-11-19T01:28:48,551 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/10d7714ea1534d16b9a707aa59fac0a0, entries=6, sequenceid=105, filesize=5.8 K 2023-11-19T01:28:48,551 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/5396f5846ade4d52893b3744b3e7409e as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/5396f5846ade4d52893b3744b3e7409e 2023-11-19T01:28:48,556 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/5396f5846ade4d52893b3744b3e7409e, entries=2, sequenceid=105, filesize=5.5 K 2023-11-19T01:28:48,556 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for 322466352cac3ade468ab2ce8e706acc in 435ms, sequenceid=105, compaction requested=true 2023-11-19T01:28:48,556 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.HRegion(2520): Flush status journal for 322466352cac3ade468ab2ce8e706acc: 2023-11-19T01:28:48,556 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. for snapshot_backup_system completed. 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/83b8d86696ce4c728d410b325dfe4099, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/817bbb8696494171be63b200e480be74, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/298b24d86eff42c285c52df630edc72c, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/e32b4b3110b04f12a891660d01b34d8c, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/a356c57983bf487eaa945a53843b801e, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/10d7714ea1534d16b9a707aa59fac0a0] hfiles 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (1/9): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757 for snapshot=snapshot_backup_system 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (2/9): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63 for snapshot=snapshot_backup_system 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (3/9): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2 for snapshot=snapshot_backup_system 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (4/9): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/83b8d86696ce4c728d410b325dfe4099 for snapshot=snapshot_backup_system 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (5/9): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/817bbb8696494171be63b200e480be74 for snapshot=snapshot_backup_system 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (6/9): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/298b24d86eff42c285c52df630edc72c for snapshot=snapshot_backup_system 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (7/9): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/e32b4b3110b04f12a891660d01b34d8c for snapshot=snapshot_backup_system 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (8/9): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/a356c57983bf487eaa945a53843b801e for snapshot=snapshot_backup_system 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (9/9): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/10d7714ea1534d16b9a707aa59fac0a0 for snapshot=snapshot_backup_system 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/05f6b05b598842a0a5af3e603f2dfa9f, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/6e39103009374e778431dfd7316f5a4c, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/1b2851533774422da3cbc0adf48207b8, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/20bd2650ab284e0b9d4e6d4318baa9b6, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/4b8411bfe401452da4354337a869a577, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/5396f5846ade4d52893b3744b3e7409e] hfiles 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (1/10): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99 for snapshot=snapshot_backup_system 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (2/10): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff for snapshot=snapshot_backup_system 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (3/10): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460 for snapshot=snapshot_backup_system 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (4/10): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1 for snapshot=snapshot_backup_system 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (5/10): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/05f6b05b598842a0a5af3e603f2dfa9f for snapshot=snapshot_backup_system 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (6/10): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/6e39103009374e778431dfd7316f5a4c for snapshot=snapshot_backup_system 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (7/10): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/1b2851533774422da3cbc0adf48207b8 for snapshot=snapshot_backup_system 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (8/10): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/20bd2650ab284e0b9d4e6d4318baa9b6 for snapshot=snapshot_backup_system 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (9/10): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/4b8411bfe401452da4354337a869a577 for snapshot=snapshot_backup_system 2023-11-19T01:28:48,557 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] snapshot.SnapshotManifest(264): Adding reference for file (10/10): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/5396f5846ade4d52893b3744b3e7409e for snapshot=snapshot_backup_system 2023-11-19T01:28:48,968 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:28:48,968 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=134 2023-11-19T01:28:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=134 2023-11-19T01:28:48,969 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:28:48,970 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=134, ppid=133, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:28:48,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=134, resume processing ppid=133 2023-11-19T01:28:48,972 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:28:48,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=134, ppid=133, state=SUCCESS; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc in 1.0030 sec 2023-11-19T01:28:48,973 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:28:48,974 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:28:48,974 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:28:48,976 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:28:49,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=133 2023-11-19T01:28:49,394 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:28:49,407 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:28:49,407 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:28:49,408 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=133, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:28:49,408 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 133 2023-11-19T01:28:49,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=133, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=133, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 1.4760 sec 2023-11-19T01:28:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=133 2023-11-19T01:28:50,047 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(115): Backup backup_1700357327885 started at 1700357330047. 2023-11-19T01:28:50,048 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(119): Backup session backup_1700357327885 has been started. 2023-11-19T01:28:50,052 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-19T01:28:50,053 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:28:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:28:50,054 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:28:50,054 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:28:50,055 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:28:50,055 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:28:50,055 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:28:50,057 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:50,057 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:28:50,057 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:28:50,057 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:28:50,057 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:28:50,058 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:28:50,058 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:28:50,058 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:28:50,058 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:28:50,058 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:28:50,058 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:28:50,058 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:28:50,060 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:28:50,060 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:28:50,060 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:28:50,065 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:28:50,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:50,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:28:50,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:28:50,065 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:28:50,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:28:50,065 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:28:50,066 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:28:50,066 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:50,066 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:28:50,067 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:28:50,067 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@554450bd[Count = 0] remaining members to acquire global barrier 2023-11-19T01:28:50,067 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:28:50,067 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:28:50,074 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:28:50,074 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:28:50,074 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:28:50,074 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:28:50,074 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:28:50,074 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:50,074 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357311068 highest: 1700357311068 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:50,074 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784:(num 1700357311068) roll requested 2023-11-19T01:28:50,086 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357330074, exclude list is [], retry=0 2023-11-19T01:28:50,090 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:28:50,091 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357311068 with entries=17, filesize=5.81 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357330074 2023-11-19T01:28:50,091 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:28:50,092 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357311068 is not closed yet, will try archiving it next time 2023-11-19T01:28:50,092 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta:.meta(num 1700357311085) roll requested 2023-11-19T01:28:50,092 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357290490 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/oldWALs/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357290490 2023-11-19T01:28:50,109 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357330092.meta, exclude list is [], retry=0 2023-11-19T01:28:50,113 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:28:50,114 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357311085.meta with entries=24, filesize=6.28 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357330092.meta 2023-11-19T01:28:50,115 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:28:50,115 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357311085.meta is not closed yet, will try archiving it next time 2023-11-19T01:28:50,134 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 60 2023-11-19T01:28:50,135 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357330074 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:50,142 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:28:50,142 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,44997,1700357176784' in zk 2023-11-19T01:28:50,145 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:28:50,145 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:28:50,145 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:28:50,146 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:50,146 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:28:50,146 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:28:50,146 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:28:50,146 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:28:50,147 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:28:50,147 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:50,147 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:28:50,147 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:28:50,148 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:50,148 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,44997,1700357176784': 2023-11-19T01:28:50,148 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:28:50,148 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:28:50,148 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:28:50,148 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:28:50,148 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:28:50,148 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:28:50,150 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:28:50,150 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:28:50,150 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:28:50,151 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:28:50,151 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:28:50,151 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:28:50,151 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:28:50,151 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:28:50,151 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:28:50,151 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:28:50,152 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:28:50,152 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:28:50,153 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:50,153 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:28:50,153 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:28:50,154 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:50,158 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:28:50,158 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:28:50,158 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:28:50,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:28:50,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:28:50,158 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:28:50,158 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:28:50,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-19T01:28:50,262 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357330261_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } 2023-11-19T01:28:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357330262 (current time:1700357330262). 2023-11-19T01:28:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:28:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357330261_ns2_test-17003571846791 VERSION not specified, setting to 2 2023-11-19T01:28:50,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@edb1300, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:50,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:50,276 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43410, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:50,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:50,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:28:50,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=135, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1700357330261_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } 2023-11-19T01:28:50,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357330261_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 }, snapshot procedure id = 135 2023-11-19T01:28:50,280 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1700357330261_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:28:50,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=135 2023-11-19T01:28:50,281 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1700357330261_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:28:50,283 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357330261_ns2_test-17003571846791 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:28:50,286 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1700357330261_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:28:50,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=135 2023-11-19T01:28:50,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=135 2023-11-19T01:28:50,691 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1700357330261_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:28:50,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE; SnapshotRegionProcedure a169e45a4f1d98e65b6a1dada440fc40}] 2023-11-19T01:28:50,693 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=136, ppid=135, state=RUNNABLE; SnapshotRegionProcedure a169e45a4f1d98e65b6a1dada440fc40 2023-11-19T01:28:50,844 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:50,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=136 2023-11-19T01:28:50,845 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40. 2023-11-19T01:28:50,845 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.HRegion(2520): Flush status journal for a169e45a4f1d98e65b6a1dada440fc40: 2023-11-19T01:28:50,845 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40. for snapshot_1700357330261_ns2_test-17003571846791 completed. 2023-11-19T01:28:50,845 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(240): Storing 'ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40.' region-info for snapshot=snapshot_1700357330261_ns2_test-17003571846791 2023-11-19T01:28:50,845 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:28:50,845 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40/f/9301f8758b1e47ab85817026d7156af0_SeqId_4_] hfiles 2023-11-19T01:28:50,845 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40/f/9301f8758b1e47ab85817026d7156af0_SeqId_4_ for snapshot=snapshot_1700357330261_ns2_test-17003571846791 2023-11-19T01:28:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=135 2023-11-19T01:28:51,251 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40. 2023-11-19T01:28:51,251 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=136 2023-11-19T01:28:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=136 2023-11-19T01:28:51,251 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357330261_ns2_test-17003571846791 on region a169e45a4f1d98e65b6a1dada440fc40 2023-11-19T01:28:51,253 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=136, ppid=135, state=RUNNABLE; SnapshotRegionProcedure a169e45a4f1d98e65b6a1dada440fc40 2023-11-19T01:28:51,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=136, resume processing ppid=135 2023-11-19T01:28:51,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=136, ppid=135, state=SUCCESS; SnapshotRegionProcedure a169e45a4f1d98e65b6a1dada440fc40 in 562 msec 2023-11-19T01:28:51,255 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1700357330261_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:28:51,255 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1700357330261_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:28:51,256 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1700357330261_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:28:51,256 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357330261_ns2_test-17003571846791 2023-11-19T01:28:51,256 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357330261_ns2_test-17003571846791 2023-11-19T01:28:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=135 2023-11-19T01:28:51,667 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1700357330261_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:28:51,672 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1700357330261_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:28:51,672 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357330261_ns2_test-17003571846791 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357330261_ns2_test-17003571846791 2023-11-19T01:28:51,673 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=135, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1700357330261_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:28:51,673 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357330261_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 }, snapshot procedure id = 135 2023-11-19T01:28:51,674 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=135, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=135, snapshot={ ss=snapshot_1700357330261_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } in 1.3960 sec 2023-11-19T01:28:52,076 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:28:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=135 2023-11-19T01:28:52,386 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357332386_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } 2023-11-19T01:28:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357332386 (current time:1700357332386). 2023-11-19T01:28:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:28:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357332386_ns3_test-17003571846792 VERSION not specified, setting to 2 2023-11-19T01:28:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d77bfd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:28:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:28:52,395 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:43426, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:28:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:28:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:28:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=137, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1700357332386_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } 2023-11-19T01:28:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357332386_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 }, snapshot procedure id = 137 2023-11-19T01:28:52,399 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1700357332386_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:28:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=137 2023-11-19T01:28:52,400 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1700357332386_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:28:52,402 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357332386_ns3_test-17003571846792 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:28:52,405 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1700357332386_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:28:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=137 2023-11-19T01:28:52,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=137 2023-11-19T01:28:52,811 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1700357332386_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:28:52,811 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE; SnapshotRegionProcedure 09936870cc2abdecec74e26057d016f7}] 2023-11-19T01:28:52,813 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=138, ppid=137, state=RUNNABLE; SnapshotRegionProcedure 09936870cc2abdecec74e26057d016f7 2023-11-19T01:28:52,964 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:28:52,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=138 2023-11-19T01:28:52,964 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7. 2023-11-19T01:28:52,966 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.HRegion(2520): Flush status journal for 09936870cc2abdecec74e26057d016f7: 2023-11-19T01:28:52,966 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7. for snapshot_1700357332386_ns3_test-17003571846792 completed. 2023-11-19T01:28:52,966 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] snapshot.SnapshotManifest(240): Storing 'ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7.' region-info for snapshot=snapshot_1700357332386_ns3_test-17003571846792 2023-11-19T01:28:52,966 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:28:52,966 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-19T01:28:53,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=137 2023-11-19T01:28:53,372 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7. 2023-11-19T01:28:53,372 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=138 2023-11-19T01:28:53,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=138 2023-11-19T01:28:53,372 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357332386_ns3_test-17003571846792 on region 09936870cc2abdecec74e26057d016f7 2023-11-19T01:28:53,374 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=138, ppid=137, state=RUNNABLE; SnapshotRegionProcedure 09936870cc2abdecec74e26057d016f7 2023-11-19T01:28:53,375 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=138, resume processing ppid=137 2023-11-19T01:28:53,375 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=138, ppid=137, state=SUCCESS; SnapshotRegionProcedure 09936870cc2abdecec74e26057d016f7 in 563 msec 2023-11-19T01:28:53,375 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1700357332386_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:28:53,376 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1700357332386_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:28:53,376 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1700357332386_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:28:53,377 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357332386_ns3_test-17003571846792 2023-11-19T01:28:53,377 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357332386_ns3_test-17003571846792 2023-11-19T01:28:53,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=137 2023-11-19T01:28:53,787 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1700357332386_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:28:53,792 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1700357332386_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:28:53,792 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357332386_ns3_test-17003571846792 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357332386_ns3_test-17003571846792 2023-11-19T01:28:53,793 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=137, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1700357332386_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:28:53,793 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357332386_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 }, snapshot procedure id = 137 2023-11-19T01:28:53,794 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=137, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=137, snapshot={ ss=snapshot_1700357332386_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } in 1.3970 sec 2023-11-19T01:28:54,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=137 2023-11-19T01:28:54,505 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1700357327885 2023-11-19T01:28:54,505 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-19T01:28:54,505 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-11-19T01:28:54,505 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357327885_ns2:test-17003571846791 2023-11-19T01:28:54,505 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357330261_ns2_test-17003571846791 to hdfs://localhost:42739/backupUT/backup_1700357327885/ns2/test-17003571846791/ 2023-11-19T01:28:54,505 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:28:54,533 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:28:54,533 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,44889,1700357173572_-1425161128_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42739/backupUT/backup_1700357327885/ns2/test-17003571846791, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42739/backupUT/backup_1700357327885/ns2/test-17003571846791/.hbase-snapshot/.tmp/snapshot_1700357330261_ns2_test-17003571846791 2023-11-19T01:28:54,533 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af, snapshotDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357330261_ns2_test-17003571846791. 2023-11-19T01:28:54,544 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357330261_ns2_test-17003571846791 to hdfs://localhost:42739/backupUT/backup_1700357327885/ns2/test-17003571846791/.hbase-snapshot/.tmp/snapshot_1700357330261_ns2_test-17003571846791 2023-11-19T01:28:54,974 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:54,975 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:54,975 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:54,976 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:56,004 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-7444668967891003421.jar 2023-11-19T01:28:56,005 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:56,005 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:56,006 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:56,007 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:56,007 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:56,007 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:56,008 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:56,008 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:56,009 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:56,010 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:28:56,011 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:28:56,011 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:28:56,012 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:28:56,013 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:28:56,013 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:28:56,014 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:28:56,014 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:28:56,015 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:28:56,015 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:28:56,016 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:28:56,017 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:28:56,018 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:56,018 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:56,019 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:56,020 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:56,020 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:56,021 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:56,022 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:56,075 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357330261_ns2_test-17003571846791' hfile list 2023-11-19T01:28:56,077 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-19T01:28:56,178 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357336129/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:28:56,178 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357336129/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:28:56,237 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-19T01:28:56,321 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40/f/9301f8758b1e47ab85817026d7156af0_SeqId_4_ output=hdfs://localhost:42739/backupUT/backup_1700357327885/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40/f/9301f8758b1e47ab85817026d7156af0_SeqId_4_ 2023-11-19T01:28:56,321 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 4.160M/sec 2023-11-19T01:28:57,211 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:28:57,212 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:28:57,220 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357330261_ns2_test-17003571846791 2023-11-19T01:28:57,221 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357330261_ns2_test-17003571846791 finished. 2023-11-19T01:28:57,221 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357327885_ns3:test-17003571846792 2023-11-19T01:28:57,221 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357332386_ns3_test-17003571846792 to hdfs://localhost:42739/backupUT/backup_1700357327885/ns3/test-17003571846792/ 2023-11-19T01:28:57,221 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:28:57,247 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:28:57,247 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,44889,1700357173572_-945426400_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42739/backupUT/backup_1700357327885/ns3/test-17003571846792, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42739/backupUT/backup_1700357327885/ns3/test-17003571846792/.hbase-snapshot/.tmp/snapshot_1700357332386_ns3_test-17003571846792 2023-11-19T01:28:57,247 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af, snapshotDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357332386_ns3_test-17003571846792. 2023-11-19T01:28:57,251 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357332386_ns3_test-17003571846792 to hdfs://localhost:42739/backupUT/backup_1700357327885/ns3/test-17003571846792/.hbase-snapshot/.tmp/snapshot_1700357332386_ns3_test-17003571846792 2023-11-19T01:28:57,671 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:57,671 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:57,672 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:57,672 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:58,701 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-3208702931773163130.jar 2023-11-19T01:28:58,701 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:58,702 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:58,702 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:58,702 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:58,703 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:58,703 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:58,703 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:58,704 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:58,704 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:28:58,705 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:28:58,705 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:28:58,706 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:28:58,706 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:28:58,707 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:28:58,707 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:28:58,707 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:28:58,708 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:28:58,708 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:28:58,709 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:28:58,709 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:28:58,709 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:28:58,710 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:58,710 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:58,711 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:58,711 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:58,711 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:28:58,712 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:58,713 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:28:58,758 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357332386_ns3_test-17003571846792' hfile list 2023-11-19T01:28:58,837 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357338799/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:28:58,837 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357338799/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:28:59,885 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:28:59,886 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:28:59,892 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357332386_ns3_test-17003571846792 2023-11-19T01:28:59,892 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357332386_ns3_test-17003571846792 finished. 2023-11-19T01:28:59,892 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1034): ns2:test-17003571846791 2023-11-19T01:28:59,892 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1034): ns3:test-17003571846792 2023-11-19T01:28:59,899 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357327885 2023-11-19T01:28:59,899 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:28:59,908 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42739/backupUT/backup_1700357327885/.backup.manifest 2023-11-19T01:28:59,908 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357327885 2023-11-19T01:28:59,908 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:29:00,315 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42739/backupUT/backup_1700357327885/.backup.manifest 2023-11-19T01:29:00,315 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(358): Backup backup_1700357327885 finished: type=FULL,tablelist=ns2:test-17003571846791;ns3:test-17003571846792,targetRootDir=hdfs://localhost:42739/backupUT,startts=1700357330047,completets=1700357339899,bytescopied=0 2023-11-19T01:29:00,315 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-19T01:29:00,315 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357330261_ns2_test-17003571846791 2023-11-19T01:29:00,316 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357330261_ns2_test-17003571846791" 2023-11-19T01:29:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357330261_ns2_test-17003571846791 2023-11-19T01:29:00,319 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357330261_ns2_test-17003571846791 for backup backup_1700357327885 succeeded. 2023-11-19T01:29:00,319 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357332386_ns3_test-17003571846792 2023-11-19T01:29:00,320 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357332386_ns3_test-17003571846792" 2023-11-19T01:29:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357332386_ns3_test-17003571846792 2023-11-19T01:29:00,322 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357332386_ns3_test-17003571846792 for backup backup_1700357327885 succeeded. 2023-11-19T01:29:00,322 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:29:00,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:29:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:29:00,328 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:29:00,329 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:29:00,359 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(376): Backup backup_1700357327885 completed. 2023-11-19T01:29:00,359 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:00,375 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@545b4fa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:00,376 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:00,377 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:49728, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:00,378 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(101): HBase table ns2:table2 does not exist. It will be created during restore process 2023-11-19T01:29:00,379 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(101): HBase table ns3:table3_restore does not exist. It will be created during restore process 2023-11-19T01:29:00,379 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357327885/.backup.manifest 2023-11-19T01:29:00,381 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357327885/.backup.manifest 2023-11-19T01:29:00,381 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357327885/.backup.manifest 2023-11-19T01:29:00,383 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357327885/.backup.manifest 2023-11-19T01:29:00,383 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357327885/.backup.manifest 2023-11-19T01:29:00,385 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357327885/.backup.manifest 2023-11-19T01:29:00,385 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(148): Restoring 'ns2:test-17003571846791' to 'ns2:table2' from full backup image hdfs://localhost:42739/backupUT/backup_1700357327885/ns2/test-17003571846791 2023-11-19T01:29:00,391 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(487): Creating target table 'ns2:table2' 2023-11-19T01:29:00,391 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:42739/backupUT/backup_1700357327885/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40 2023-11-19T01:29:00,392 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:42739/backupUT/backup_1700357327885/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40/f in region [hdfs://localhost:42739/backupUT/backup_1700357327885/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40] 2023-11-19T01:29:00,394 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:42739/backupUT/backup_1700357327885/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40/f/9301f8758b1e47ab85817026d7156af0_SeqId_4_ first=row0 last=row98 2023-11-19T01:29:00,395 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:00,397 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:56488, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:29:00,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:29:00,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=139, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns2:table2 2023-11-19T01:29:00,400 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:29:00,401 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "ns2" qualifier: "table2" procId is: 139 2023-11-19T01:29:00,401 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:29:00,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=139 2023-11-19T01:29:00,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=139 2023-11-19T01:29:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=139 2023-11-19T01:29:00,808 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => c8fed052d135fce145e248f08097980a, NAME => 'ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns2:table2', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:29:01,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=139 2023-11-19T01:29:01,221 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(888): Instantiated ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:01,221 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1668): Closing c8fed052d135fce145e248f08097980a, disabling compactions & flushes 2023-11-19T01:29:01,221 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1690): Closing region ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a. 2023-11-19T01:29:01,222 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a. 2023-11-19T01:29:01,222 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a. after waiting 0 ms 2023-11-19T01:29:01,222 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a. 2023-11-19T01:29:01,222 INFO [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1909): Closed ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a. 2023-11-19T01:29:01,222 DEBUG [RegionOpenAndInit-ns2:table2-pool-0 {}] regionserver.HRegion(1622): Region close journal for c8fed052d135fce145e248f08097980a: 2023-11-19T01:29:01,223 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:29:01,223 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a.","families":{"info":[{"qualifier":"regioninfo","vlen":36,"tag":[],"timestamp":"1700357341223"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357341223"}]},"ts":"1700357341223"} 2023-11-19T01:29:01,224 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:29:01,225 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:29:01,225 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357341225"}]},"ts":"1700357341225"} 2023-11-19T01:29:01,226 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLING in hbase:meta 2023-11-19T01:29:01,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=c8fed052d135fce145e248f08097980a, ASSIGN}] 2023-11-19T01:29:01,236 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns2:table2, region=c8fed052d135fce145e248f08097980a, ASSIGN 2023-11-19T01:29:01,236 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(262): Starting pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns2:table2, region=c8fed052d135fce145e248f08097980a, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,44997,1700357176784; forceNewPlan=false, retain=false 2023-11-19T01:29:01,388 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=140 updating hbase:meta row=c8fed052d135fce145e248f08097980a, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:01,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE; OpenRegionProcedure c8fed052d135fce145e248f08097980a, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:29:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=139 2023-11-19T01:29:01,542 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:01,545 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(130): Open ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a. 2023-11-19T01:29:01,545 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7254): Opening region: {ENCODED => c8fed052d135fce145e248f08097980a, NAME => 'ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:29:01,545 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:29:01,546 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table2 c8fed052d135fce145e248f08097980a 2023-11-19T01:29:01,546 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(888): Instantiated ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:01,546 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7296): checking encryption for c8fed052d135fce145e248f08097980a 2023-11-19T01:29:01,546 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7299): checking classloading for c8fed052d135fce145e248f08097980a 2023-11-19T01:29:01,547 INFO [StoreOpener-c8fed052d135fce145e248f08097980a-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region c8fed052d135fce145e248f08097980a 2023-11-19T01:29:01,549 INFO [StoreOpener-c8fed052d135fce145e248f08097980a-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8fed052d135fce145e248f08097980a columnFamilyName f 2023-11-19T01:29:01,549 DEBUG [StoreOpener-c8fed052d135fce145e248f08097980a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:01,549 INFO [StoreOpener-c8fed052d135fce145e248f08097980a-1 {}] regionserver.HStore(324): Store=c8fed052d135fce145e248f08097980a/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:29:01,550 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/c8fed052d135fce145e248f08097980a 2023-11-19T01:29:01,550 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/c8fed052d135fce145e248f08097980a 2023-11-19T01:29:01,553 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1076): writing seq id for c8fed052d135fce145e248f08097980a 2023-11-19T01:29:01,554 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/c8fed052d135fce145e248f08097980a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:29:01,555 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1093): Opened c8fed052d135fce145e248f08097980a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=11173140960, jitterRate=0.040579840540885925}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:29:01,556 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(992): Region open journal for c8fed052d135fce145e248f08097980a: 2023-11-19T01:29:01,556 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2556): Post open deploy tasks for ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a., pid=141, masterSystemTime=1700357341541 2023-11-19T01:29:01,558 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2583): Finished post open deploy task for ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a. 2023-11-19T01:29:01,558 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(158): Opened ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a. 2023-11-19T01:29:01,558 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=140 updating hbase:meta row=c8fed052d135fce145e248f08097980a, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:01,561 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=141, resume processing ppid=140 2023-11-19T01:29:01,561 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=141, ppid=140, state=SUCCESS; OpenRegionProcedure c8fed052d135fce145e248f08097980a, server=jenkins-hbase6.apache.org,44997,1700357176784 in 170 msec 2023-11-19T01:29:01,563 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=140, resume processing ppid=139 2023-11-19T01:29:01,563 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=140, ppid=139, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=c8fed052d135fce145e248f08097980a, ASSIGN in 327 msec 2023-11-19T01:29:01,563 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:29:01,563 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357341563"}]},"ts":"1700357341563"} 2023-11-19T01:29:01,564 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=ENABLED in hbase:meta 2023-11-19T01:29:01,567 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(81): pid=139, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns2:table2 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:29:01,568 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=139, state=SUCCESS; CreateTableProcedure table=ns2:table2 in 1.1690 sec 2023-11-19T01:29:02,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=139 2023-11-19T01:29:02,514 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:42739/backupUT/backup_1700357327885/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40 from hbase tables ns2:test-17003571846791 to tables ns2:table2 2023-11-19T01:29:02,514 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(72): Restore ns2:test-17003571846791 into ns2:table2 2023-11-19T01:29:02,516 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-table2-1700357342514 from hdfs://localhost:42739/backupUT/backup_1700357327885/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40 2023-11-19T01:29:02,521 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45654009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:02,522 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:02,522 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:56504, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:29:02,523 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-19T01:29:02,523 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table ns2:table2 2023-11-19T01:29:02,528 DEBUG [hconnection-0xe971c81-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:02,529 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:49740, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:02,530 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for ns2:table2: 2023-11-19T01:29:02,530 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-11-19T01:29:02,530 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/partitions_a764d4c7-3597-43f8-961c-8dbc56f5730d 2023-11-19T01:29:02,937 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:02,937 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:02,938 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:02,938 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:03,934 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-4081037243732620254.jar 2023-11-19T01:29:03,934 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:03,934 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:03,935 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:03,935 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:03,935 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:03,936 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:03,936 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:03,936 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:03,937 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:03,937 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:29:03,938 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:29:03,938 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:29:03,939 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:29:03,939 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:29:03,940 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:29:03,940 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:29:03,940 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:29:03,941 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:29:03,941 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:29:03,942 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:29:03,942 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:29:03,942 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:03,943 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:03,943 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:03,943 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:03,943 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:03,944 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:03,944 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:03,945 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: ns2:table2 2023-11-19T01:29:03,945 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:03,945 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-11-19T01:29:03,945 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-11-19T01:29:03,946 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:29:03,958 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-11-19T01:29:04,033 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357344001/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:29:04,033 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357344001/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:29:04,081 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:42739/backupUT/backup_1700357327885/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40/f/9301f8758b1e47ab85817026d7156af0_SeqId_4_ 2023-11-19T01:29:04,084 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-11-19T01:29:04,106 DEBUG [pool-485-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4db90f80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:04,107 DEBUG [hconnection-0x5f8a4fd6-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:04,109 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:49752, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:04,112 DEBUG [pool-485-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:04,112 DEBUG [pool-485-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-11-19T01:29:04,112 DEBUG [pool-485-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase6.apache.org 2023-11-19T01:29:04,117 DEBUG [pool-485-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-table2-1700357342514/_temporary/0/_temporary/attempt_local1350286002_0022_r_000000_0/f/34d8a81c9f47452089856472eda987f3 is 37, key is row10/f:q1/1700357188531/Put/seqid=0 2023-11-19T01:29:04,118 WARN [Thread-2116 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase6.apache.org:44997] Specified favored nodes: [jenkins-hbase6.apache.org:44997] 2023-11-19T01:29:04,784 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2023-11-19T01:29:04,784 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_ns2_table_table2 Metrics about Tables on a single HBase RegionServer 2023-11-19T01:29:05,081 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-table2-1700357342514 2023-11-19T01:29:05,088 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15cd2d38, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:05,089 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:05,091 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:49756, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:05,097 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-table2-1700357342514/_SUCCESS 2023-11-19T01:29:05,101 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:05,103 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:56518, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:29:05,115 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-table2-1700357342514/f/34d8a81c9f47452089856472eda987f3 first=Optional[row0] last=Optional[row98] 2023-11-19T01:29:05,116 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a., hostname=jenkins-hbase6.apache.org,44997,1700357176784, seqNum=2 for row with hfile group [{f,hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-table2-1700357342514/f/34d8a81c9f47452089856472eda987f3}] 2023-11-19T01:29:05,121 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-table2-1700357342514/f/34d8a81c9f47452089856472eda987f3 for inclusion in c8fed052d135fce145e248f08097980a/f 2023-11-19T01:29:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-11-19T01:29:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(625): Region bounds: first= last= 2023-11-19T01:29:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegion(2520): Flush status journal for c8fed052d135fce145e248f08097980a: 2023-11-19T01:29:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-ns2-table2-1700357342514/f/34d8a81c9f47452089856472eda987f3 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__ns2_table2__a08ngo61gibajlacrinvoh3nmol6td0k19qpt2i2lpsdkhc9grcbjh3jf9gsd8pm/f/34d8a81c9f47452089856472eda987f3 2023-11-19T01:29:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__ns2_table2__a08ngo61gibajlacrinvoh3nmol6td0k19qpt2i2lpsdkhc9grcbjh3jf9gsd8pm/f/34d8a81c9f47452089856472eda987f3 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/c8fed052d135fce145e248f08097980a/f/103e2a7b8d3b4800ba4aec4bbd205d4a_SeqId_4_ 2023-11-19T01:29:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@317a9e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:05,135 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:56532, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:29:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:05,139 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:49766, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:05,150 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__ns2_table2__a08ngo61gibajlacrinvoh3nmol6td0k19qpt2i2lpsdkhc9grcbjh3jf9gsd8pm/f/34d8a81c9f47452089856472eda987f3 into c8fed052d135fce145e248f08097980a/f as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/c8fed052d135fce145e248f08097980a/f/103e2a7b8d3b4800ba4aec4bbd205d4a_SeqId_4_ - updating store file list. 2023-11-19T01:29:05,155 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/c8fed052d135fce145e248f08097980a/f/103e2a7b8d3b4800ba4aec4bbd205d4a_SeqId_4_ into c8fed052d135fce145e248f08097980a/f 2023-11-19T01:29:05,155 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__ns2_table2__a08ngo61gibajlacrinvoh3nmol6td0k19qpt2i2lpsdkhc9grcbjh3jf9gsd8pm/f/34d8a81c9f47452089856472eda987f3 into c8fed052d135fce145e248f08097980a/f (new location: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/c8fed052d135fce145e248f08097980a/f/103e2a7b8d3b4800ba4aec4bbd205d4a_SeqId_4_) 2023-11-19T01:29:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__ns2_table2__a08ngo61gibajlacrinvoh3nmol6td0k19qpt2i2lpsdkhc9grcbjh3jf9gsd8pm/f/34d8a81c9f47452089856472eda987f3 2023-11-19T01:29:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@179b071c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:05,164 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:56544, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-19T01:29:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:05,168 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:49772, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-11-19T01:29:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:05,181 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:05,182 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-11-19T01:29:05,182 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-19T01:29:05,182 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(233): Backup: backup_1700357327885 hdfs://localhost:42739/backupUT/backup_1700357327885/ns2/test-17003571846791/ 2023-11-19T01:29:05,182 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357327885/.backup.manifest 2023-11-19T01:29:05,186 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357327885/.backup.manifest 2023-11-19T01:29:05,186 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(148): Restoring 'ns3:test-17003571846792' to 'ns3:table3_restore' from full backup image hdfs://localhost:42739/backupUT/backup_1700357327885/ns3/test-17003571846792 2023-11-19T01:29:05,191 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(101): Folder tableArchivePath: hdfs://localhost:42739/backupUT/backup_1700357327885/ns3/test-17003571846792/archive/data/ns3/test-17003571846792 does not exists 2023-11-19T01:29:05,191 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(326): find table descriptor but no archive dir for table ns3:test-17003571846792, will only create table 2023-11-19T01:29:05,193 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(487): Creating target table 'ns3:table3_restore' 2023-11-19T01:29:05,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:29:05,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=ns3:table3_restore 2023-11-19T01:29:05,196 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:29:05,196 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "ns3" qualifier: "table3_restore" procId is: 142 2023-11-19T01:29:05,196 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:29:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=142 2023-11-19T01:29:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=142 2023-11-19T01:29:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=142 2023-11-19T01:29:05,603 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => 94b611c407db66407817d625f99b6d97, NAME => 'ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='ns3:table3_restore', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:29:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=142 2023-11-19T01:29:06,009 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:06,009 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1668): Closing 94b611c407db66407817d625f99b6d97, disabling compactions & flushes 2023-11-19T01:29:06,009 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97. 2023-11-19T01:29:06,009 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97. 2023-11-19T01:29:06,009 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97. after waiting 0 ms 2023-11-19T01:29:06,009 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97. 2023-11-19T01:29:06,009 INFO [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97. 2023-11-19T01:29:06,009 DEBUG [RegionOpenAndInit-ns3:table3_restore-pool-0 {}] regionserver.HRegion(1622): Region close journal for 94b611c407db66407817d625f99b6d97: 2023-11-19T01:29:06,010 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:29:06,010 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1700357346010"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357346010"}]},"ts":"1700357346010"} 2023-11-19T01:29:06,011 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:29:06,012 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:29:06,012 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357346012"}]},"ts":"1700357346012"} 2023-11-19T01:29:06,013 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLING in hbase:meta 2023-11-19T01:29:06,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=94b611c407db66407817d625f99b6d97, ASSIGN}] 2023-11-19T01:29:06,020 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=ns3:table3_restore, region=94b611c407db66407817d625f99b6d97, ASSIGN 2023-11-19T01:29:06,020 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(262): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=ns3:table3_restore, region=94b611c407db66407817d625f99b6d97, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,44997,1700357176784; forceNewPlan=false, retain=false 2023-11-19T01:29:06,172 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=143 updating hbase:meta row=94b611c407db66407817d625f99b6d97, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:06,173 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE; OpenRegionProcedure 94b611c407db66407817d625f99b6d97, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:29:06,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=142 2023-11-19T01:29:06,325 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:06,328 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(130): Open ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97. 2023-11-19T01:29:06,328 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7254): Opening region: {ENCODED => 94b611c407db66407817d625f99b6d97, NAME => 'ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:29:06,328 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:29:06,328 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table3_restore 94b611c407db66407817d625f99b6d97 2023-11-19T01:29:06,328 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(888): Instantiated ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:06,328 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7296): checking encryption for 94b611c407db66407817d625f99b6d97 2023-11-19T01:29:06,328 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7299): checking classloading for 94b611c407db66407817d625f99b6d97 2023-11-19T01:29:06,330 INFO [StoreOpener-94b611c407db66407817d625f99b6d97-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region 94b611c407db66407817d625f99b6d97 2023-11-19T01:29:06,331 INFO [StoreOpener-94b611c407db66407817d625f99b6d97-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94b611c407db66407817d625f99b6d97 columnFamilyName f 2023-11-19T01:29:06,331 DEBUG [StoreOpener-94b611c407db66407817d625f99b6d97-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:06,331 INFO [StoreOpener-94b611c407db66407817d625f99b6d97-1 {}] regionserver.HStore(324): Store=94b611c407db66407817d625f99b6d97/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:29:06,332 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/table3_restore/94b611c407db66407817d625f99b6d97 2023-11-19T01:29:06,332 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/table3_restore/94b611c407db66407817d625f99b6d97 2023-11-19T01:29:06,334 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1076): writing seq id for 94b611c407db66407817d625f99b6d97 2023-11-19T01:29:06,336 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/table3_restore/94b611c407db66407817d625f99b6d97/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:29:06,336 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): Opened 94b611c407db66407817d625f99b6d97; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=10199868480, jitterRate=-0.05006322264671326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:29:06,336 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(992): Region open journal for 94b611c407db66407817d625f99b6d97: 2023-11-19T01:29:06,337 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2556): Post open deploy tasks for ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97., pid=144, masterSystemTime=1700357346324 2023-11-19T01:29:06,338 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2583): Finished post open deploy task for ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97. 2023-11-19T01:29:06,338 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(158): Opened ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97. 2023-11-19T01:29:06,339 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=143 updating hbase:meta row=94b611c407db66407817d625f99b6d97, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:06,340 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=144, resume processing ppid=143 2023-11-19T01:29:06,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=144, ppid=143, state=SUCCESS; OpenRegionProcedure 94b611c407db66407817d625f99b6d97, server=jenkins-hbase6.apache.org,44997,1700357176784 in 167 msec 2023-11-19T01:29:06,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=143, resume processing ppid=142 2023-11-19T01:29:06,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=143, ppid=142, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=94b611c407db66407817d625f99b6d97, ASSIGN in 323 msec 2023-11-19T01:29:06,342 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:29:06,342 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357346342"}]},"ts":"1700357346342"} 2023-11-19T01:29:06,343 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=ENABLED in hbase:meta 2023-11-19T01:29:06,346 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(81): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=ns3:table3_restore execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:29:06,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=142, state=SUCCESS; CreateTableProcedure table=ns3:table3_restore in 1.1520 sec 2023-11-19T01:29:07,090 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:29:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=142 2023-11-19T01:29:07,304 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-19T01:29:07,304 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(233): Backup: backup_1700357327885 hdfs://localhost:42739/backupUT/backup_1700357327885/ns3/test-17003571846792/ 2023-11-19T01:29:07,304 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-11-19T01:29:07,304 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:07,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable ns2:table2 2023-11-19T01:29:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=145, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns2:table2 2023-11-19T01:29:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=145 2023-11-19T01:29:07,311 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357347310"}]},"ts":"1700357347310"} 2023-11-19T01:29:07,311 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLING in hbase:meta 2023-11-19T01:29:07,313 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(293): Set ns2:table2 to state=DISABLING 2023-11-19T01:29:07,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=c8fed052d135fce145e248f08097980a, UNASSIGN}] 2023-11-19T01:29:07,315 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=146, ppid=145, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns2:table2, region=c8fed052d135fce145e248f08097980a, UNASSIGN 2023-11-19T01:29:07,315 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=146 updating hbase:meta row=c8fed052d135fce145e248f08097980a, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:07,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE; CloseRegionProcedure c8fed052d135fce145e248f08097980a, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:29:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=145 2023-11-19T01:29:07,467 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:07,468 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] handler.UnassignRegionHandler(119): Close c8fed052d135fce145e248f08097980a 2023-11-19T01:29:07,468 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1668): Closing c8fed052d135fce145e248f08097980a, disabling compactions & flushes 2023-11-19T01:29:07,468 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1690): Closing region ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a. 2023-11-19T01:29:07,468 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a. 2023-11-19T01:29:07,468 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1778): Acquired close lock on ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a. after waiting 0 ms 2023-11-19T01:29:07,468 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1788): Updates disabled for region ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a. 2023-11-19T01:29:07,473 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/c8fed052d135fce145e248f08097980a/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-11-19T01:29:07,474 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:29:07,474 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1909): Closed ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a. 2023-11-19T01:29:07,474 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] regionserver.HRegion(1622): Region close journal for c8fed052d135fce145e248f08097980a: 2023-11-19T01:29:07,475 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=147}] handler.UnassignRegionHandler(163): Closed c8fed052d135fce145e248f08097980a 2023-11-19T01:29:07,476 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=146 updating hbase:meta row=c8fed052d135fce145e248f08097980a, regionState=CLOSED 2023-11-19T01:29:07,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=147, resume processing ppid=146 2023-11-19T01:29:07,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=147, ppid=146, state=SUCCESS; CloseRegionProcedure c8fed052d135fce145e248f08097980a, server=jenkins-hbase6.apache.org,44997,1700357176784 in 161 msec 2023-11-19T01:29:07,479 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=146, resume processing ppid=145 2023-11-19T01:29:07,479 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=146, ppid=145, state=SUCCESS; TransitRegionStateProcedure table=ns2:table2, region=c8fed052d135fce145e248f08097980a, UNASSIGN in 165 msec 2023-11-19T01:29:07,479 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357347479"}]},"ts":"1700357347479"} 2023-11-19T01:29:07,480 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns2:table2, state=DISABLED in hbase:meta 2023-11-19T01:29:07,483 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(305): Set ns2:table2 to state=DISABLED 2023-11-19T01:29:07,484 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=145, state=SUCCESS; DisableTableProcedure table=ns2:table2 in 176 msec 2023-11-19T01:29:07,489 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'ns3:table3_restore' 2023-11-19T01:29:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=145 2023-11-19T01:29:07,613 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$5(2449): Client=jenkins//172.31.2.20 delete ns2:table2 2023-11-19T01:29:07,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=148, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns2:table2 2023-11-19T01:29:07,615 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=148, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-11-19T01:29:07,616 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=148, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns2:table2 2023-11-19T01:29:07,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=148 2023-11-19T01:29:07,618 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/c8fed052d135fce145e248f08097980a 2023-11-19T01:29:07,619 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/c8fed052d135fce145e248f08097980a/f, FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/c8fed052d135fce145e248f08097980a/recovered.edits] 2023-11-19T01:29:07,623 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/c8fed052d135fce145e248f08097980a/f/103e2a7b8d3b4800ba4aec4bbd205d4a_SeqId_4_ to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/archive/data/ns2/table2/c8fed052d135fce145e248f08097980a/f/103e2a7b8d3b4800ba4aec4bbd205d4a_SeqId_4_ 2023-11-19T01:29:07,626 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/c8fed052d135fce145e248f08097980a/recovered.edits/6.seqid to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/archive/data/ns2/table2/c8fed052d135fce145e248f08097980a/recovered.edits/6.seqid 2023-11-19T01:29:07,626 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/table2/c8fed052d135fce145e248f08097980a 2023-11-19T01:29:07,626 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(293): Archived ns2:table2 regions 2023-11-19T01:29:07,628 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=148, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns2:table2 2023-11-19T01:29:07,630 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns2:table2 from hbase:meta 2023-11-19T01:29:07,631 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(387): Removing 'ns2:table2' descriptor. 2023-11-19T01:29:07,632 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=148, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns2:table2 2023-11-19T01:29:07,632 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(377): Removing 'ns2:table2' from region states. 2023-11-19T01:29:07,632 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357347632"}]},"ts":"9223372036854775807"} 2023-11-19T01:29:07,634 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:29:07,634 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => c8fed052d135fce145e248f08097980a, NAME => 'ns2:table2,,1700357340397.c8fed052d135fce145e248f08097980a.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:29:07,634 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(381): Marking 'ns2:table2' as deleted. 2023-11-19T01:29:07,634 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns2:table2","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1700357347634"}]},"ts":"9223372036854775807"} 2023-11-19T01:29:07,635 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1678): Deleted table ns2:table2 state from META 2023-11-19T01:29:07,637 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(127): Finished pid=148, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns2:table2 2023-11-19T01:29:07,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=148, state=SUCCESS; DeleteTableProcedure table=ns2:table2 in 24 msec 2023-11-19T01:29:07,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=148 2023-11-19T01:29:07,718 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable ns3:table3_restore 2023-11-19T01:29:07,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=149, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=ns3:table3_restore 2023-11-19T01:29:07,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=149 2023-11-19T01:29:07,721 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357347721"}]},"ts":"1700357347721"} 2023-11-19T01:29:07,722 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLING in hbase:meta 2023-11-19T01:29:07,724 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set ns3:table3_restore to state=DISABLING 2023-11-19T01:29:07,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=94b611c407db66407817d625f99b6d97, UNASSIGN}] 2023-11-19T01:29:07,726 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=150, ppid=149, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=ns3:table3_restore, region=94b611c407db66407817d625f99b6d97, UNASSIGN 2023-11-19T01:29:07,726 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=150 updating hbase:meta row=94b611c407db66407817d625f99b6d97, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:07,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE; CloseRegionProcedure 94b611c407db66407817d625f99b6d97, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:29:07,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=149 2023-11-19T01:29:07,878 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:07,879 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] handler.UnassignRegionHandler(119): Close 94b611c407db66407817d625f99b6d97 2023-11-19T01:29:07,880 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1668): Closing 94b611c407db66407817d625f99b6d97, disabling compactions & flushes 2023-11-19T01:29:07,880 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1690): Closing region ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97. 2023-11-19T01:29:07,880 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97. 2023-11-19T01:29:07,880 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1778): Acquired close lock on ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97. after waiting 0 ms 2023-11-19T01:29:07,880 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1788): Updates disabled for region ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97. 2023-11-19T01:29:07,883 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/table3_restore/94b611c407db66407817d625f99b6d97/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-19T01:29:07,884 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:29:07,885 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1909): Closed ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97. 2023-11-19T01:29:07,885 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] regionserver.HRegion(1622): Region close journal for 94b611c407db66407817d625f99b6d97: 2023-11-19T01:29:07,886 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=151}] handler.UnassignRegionHandler(163): Closed 94b611c407db66407817d625f99b6d97 2023-11-19T01:29:07,886 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=150 updating hbase:meta row=94b611c407db66407817d625f99b6d97, regionState=CLOSED 2023-11-19T01:29:07,888 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=151, resume processing ppid=150 2023-11-19T01:29:07,888 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=151, ppid=150, state=SUCCESS; CloseRegionProcedure 94b611c407db66407817d625f99b6d97, server=jenkins-hbase6.apache.org,44997,1700357176784 in 160 msec 2023-11-19T01:29:07,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=150, resume processing ppid=149 2023-11-19T01:29:07,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=150, ppid=149, state=SUCCESS; TransitRegionStateProcedure table=ns3:table3_restore, region=94b611c407db66407817d625f99b6d97, UNASSIGN in 164 msec 2023-11-19T01:29:07,890 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357347890"}]},"ts":"1700357347890"} 2023-11-19T01:29:07,891 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=ns3:table3_restore, state=DISABLED in hbase:meta 2023-11-19T01:29:07,892 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set ns3:table3_restore to state=DISABLED 2023-11-19T01:29:07,893 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=149, state=SUCCESS; DisableTableProcedure table=ns3:table3_restore in 175 msec 2023-11-19T01:29:08,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=149 2023-11-19T01:29:08,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$5(2449): Client=jenkins//172.31.2.20 delete ns3:table3_restore 2023-11-19T01:29:08,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=152, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=ns3:table3_restore 2023-11-19T01:29:08,026 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=152, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-11-19T01:29:08,027 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=152, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-11-19T01:29:08,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=152 2023-11-19T01:29:08,028 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/table3_restore/94b611c407db66407817d625f99b6d97 2023-11-19T01:29:08,030 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/table3_restore/94b611c407db66407817d625f99b6d97/f, FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/table3_restore/94b611c407db66407817d625f99b6d97/recovered.edits] 2023-11-19T01:29:08,034 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/table3_restore/94b611c407db66407817d625f99b6d97/recovered.edits/4.seqid to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/archive/data/ns3/table3_restore/94b611c407db66407817d625f99b6d97/recovered.edits/4.seqid 2023-11-19T01:29:08,034 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/table3_restore/94b611c407db66407817d625f99b6d97 2023-11-19T01:29:08,035 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived ns3:table3_restore regions 2023-11-19T01:29:08,036 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=152, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-11-19T01:29:08,038 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of ns3:table3_restore from hbase:meta 2023-11-19T01:29:08,039 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'ns3:table3_restore' descriptor. 2023-11-19T01:29:08,040 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=152, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-11-19T01:29:08,040 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'ns3:table3_restore' from region states. 2023-11-19T01:29:08,040 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357348040"}]},"ts":"9223372036854775807"} 2023-11-19T01:29:08,041 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:29:08,041 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 94b611c407db66407817d625f99b6d97, NAME => 'ns3:table3_restore,,1700357345194.94b611c407db66407817d625f99b6d97.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:29:08,041 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'ns3:table3_restore' as deleted. 2023-11-19T01:29:08,041 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"ns3:table3_restore","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1700357348041"}]},"ts":"9223372036854775807"} 2023-11-19T01:29:08,043 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table ns3:table3_restore state from META 2023-11-19T01:29:08,045 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(127): Finished pid=152, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=ns3:table3_restore 2023-11-19T01:29:08,045 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=152, state=SUCCESS; DeleteTableProcedure table=ns3:table3_restore in 21 msec 2023-11-19T01:29:08,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=152 2023-11-19T01:29:08,147 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleCommand Thread=605 (was 607), OpenFileDescriptor=708 (was 712), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=124 (was 104) - SystemLoadAverage LEAK? -, ProcessCount=166 (was 166), AvailableMemoryMB=3588 (was 4423) 2023-11-19T01:29:08,147 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-11-19T01:29:08,163 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleDNE Thread=605, OpenFileDescriptor=708, MaxFileDescriptor=60000, SystemLoadAverage=124, ProcessCount=166, AvailableMemoryMB=3588 2023-11-19T01:29:08,163 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-11-19T01:29:08,164 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestFullRestore(260): test restore fails on a single table that does not exist 2023-11-19T01:29:08,169 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a7b3ff4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:08,171 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:08,173 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:49788, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:08,174 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:08,175 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:56558, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:29:08,185 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:29:08,190 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:08,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357348190 (current time:1700357348190). 2023-11-19T01:29:08,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:29:08,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:29:08,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b9e66f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:08,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:08,197 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:49796, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:08,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:08,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:29:08,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=153, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:08,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 153 2023-11-19T01:29:08,201 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:29:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=153 2023-11-19T01:29:08,201 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:29:08,203 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:29:08,206 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:29:08,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=153 2023-11-19T01:29:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=153 2023-11-19T01:29:08,611 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:29:08,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc}] 2023-11-19T01:29:08,613 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=154, ppid=153, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:29:08,763 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:08,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=154 2023-11-19T01:29:08,764 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:29:08,764 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegion(2819): Flushing 322466352cac3ade468ab2ce8e706acc 2/2 column families, dataSize=1.50 KB heapSize=2.73 KB 2023-11-19T01:29:08,767 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/5dd4a91274764dc0b74a4aac38eaae51 is 171, key is trslm:hdfs://localhost:42739/backupUT\x00ns2:test-17003571846791/meta:log-roll-map/1700357339895/Put/seqid=0 2023-11-19T01:29:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=153 2023-11-19T01:29:09,172 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/5dd4a91274764dc0b74a4aac38eaae51 2023-11-19T01:29:09,180 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/dea7ca42cee94103a53d6e0df4cd47bd is 524, key is session:backup_1700357327885/session:context/1700357340328/Put/seqid=0 2023-11-19T01:29:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=153 2023-11-19T01:29:09,584 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=820 B at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/dea7ca42cee94103a53d6e0df4cd47bd 2023-11-19T01:29:09,590 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/5dd4a91274764dc0b74a4aac38eaae51 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5dd4a91274764dc0b74a4aac38eaae51 2023-11-19T01:29:09,594 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5dd4a91274764dc0b74a4aac38eaae51, entries=6, sequenceid=116, filesize=5.8 K 2023-11-19T01:29:09,594 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/dea7ca42cee94103a53d6e0df4cd47bd as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/dea7ca42cee94103a53d6e0df4cd47bd 2023-11-19T01:29:09,599 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/dea7ca42cee94103a53d6e0df4cd47bd, entries=2, sequenceid=116, filesize=5.5 K 2023-11-19T01:29:09,600 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegion(3022): Finished flush of dataSize ~1.50 KB/1537, heapSize ~2.70 KB/2760, currentSize=0 B/0 for 322466352cac3ade468ab2ce8e706acc in 836ms, sequenceid=116, compaction requested=true 2023-11-19T01:29:09,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.HRegion(2520): Flush status journal for 322466352cac3ade468ab2ce8e706acc: 2023-11-19T01:29:09,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. for snapshot_backup_system completed. 2023-11-19T01:29:09,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:29:09,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:29:09,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/83b8d86696ce4c728d410b325dfe4099, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/817bbb8696494171be63b200e480be74, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/298b24d86eff42c285c52df630edc72c, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/e32b4b3110b04f12a891660d01b34d8c, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/a356c57983bf487eaa945a53843b801e, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/10d7714ea1534d16b9a707aa59fac0a0, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5dd4a91274764dc0b74a4aac38eaae51] hfiles 2023-11-19T01:29:09,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (1/10): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757 for snapshot=snapshot_backup_system 2023-11-19T01:29:09,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (2/10): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63 for snapshot=snapshot_backup_system 2023-11-19T01:29:09,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (3/10): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2 for snapshot=snapshot_backup_system 2023-11-19T01:29:09,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (4/10): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/83b8d86696ce4c728d410b325dfe4099 for snapshot=snapshot_backup_system 2023-11-19T01:29:09,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (5/10): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/817bbb8696494171be63b200e480be74 for snapshot=snapshot_backup_system 2023-11-19T01:29:09,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (6/10): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/298b24d86eff42c285c52df630edc72c for snapshot=snapshot_backup_system 2023-11-19T01:29:09,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (7/10): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/e32b4b3110b04f12a891660d01b34d8c for snapshot=snapshot_backup_system 2023-11-19T01:29:09,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (8/10): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/a356c57983bf487eaa945a53843b801e for snapshot=snapshot_backup_system 2023-11-19T01:29:09,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (9/10): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/10d7714ea1534d16b9a707aa59fac0a0 for snapshot=snapshot_backup_system 2023-11-19T01:29:09,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (10/10): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5dd4a91274764dc0b74a4aac38eaae51 for snapshot=snapshot_backup_system 2023-11-19T01:29:09,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/05f6b05b598842a0a5af3e603f2dfa9f, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/6e39103009374e778431dfd7316f5a4c, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/1b2851533774422da3cbc0adf48207b8, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/20bd2650ab284e0b9d4e6d4318baa9b6, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/4b8411bfe401452da4354337a869a577, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/5396f5846ade4d52893b3744b3e7409e, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/dea7ca42cee94103a53d6e0df4cd47bd] hfiles 2023-11-19T01:29:09,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (1/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99 for snapshot=snapshot_backup_system 2023-11-19T01:29:09,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (2/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff for snapshot=snapshot_backup_system 2023-11-19T01:29:09,600 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (3/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460 for snapshot=snapshot_backup_system 2023-11-19T01:29:09,601 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (4/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1 for snapshot=snapshot_backup_system 2023-11-19T01:29:09,601 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (5/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/05f6b05b598842a0a5af3e603f2dfa9f for snapshot=snapshot_backup_system 2023-11-19T01:29:09,601 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (6/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/6e39103009374e778431dfd7316f5a4c for snapshot=snapshot_backup_system 2023-11-19T01:29:09,601 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (7/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/1b2851533774422da3cbc0adf48207b8 for snapshot=snapshot_backup_system 2023-11-19T01:29:09,601 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (8/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/20bd2650ab284e0b9d4e6d4318baa9b6 for snapshot=snapshot_backup_system 2023-11-19T01:29:09,601 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (9/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/4b8411bfe401452da4354337a869a577 for snapshot=snapshot_backup_system 2023-11-19T01:29:09,601 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (10/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/5396f5846ade4d52893b3744b3e7409e for snapshot=snapshot_backup_system 2023-11-19T01:29:09,601 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] snapshot.SnapshotManifest(264): Adding reference for file (11/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/dea7ca42cee94103a53d6e0df4cd47bd for snapshot=snapshot_backup_system 2023-11-19T01:29:10,005 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:29:10,006 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=154}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=154 2023-11-19T01:29:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=154 2023-11-19T01:29:10,006 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:29:10,007 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=154, ppid=153, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:29:10,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=154, resume processing ppid=153 2023-11-19T01:29:10,009 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:29:10,009 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=154, ppid=153, state=SUCCESS; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc in 1.3970 sec 2023-11-19T01:29:10,010 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:29:10,010 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:29:10,010 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:29:10,011 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:29:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=153 2023-11-19T01:29:10,422 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:29:10,435 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:29:10,435 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:29:10,436 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=153, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:29:10,436 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 153 2023-11-19T01:29:10,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=153, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=153, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2380 sec 2023-11-19T01:29:12,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=153 2023-11-19T01:29:12,306 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(115): Backup backup_1700357348169 started at 1700357352306. 2023-11-19T01:29:12,308 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(119): Backup session backup_1700357348169 has been started. 2023-11-19T01:29:12,314 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-19T01:29:12,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:29:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:29:12,315 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:29:12,315 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:29:12,316 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:29:12,316 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:29:12,317 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:29:12,318 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:12,318 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:29:12,318 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:29:12,319 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:29:12,319 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:29:12,319 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:29:12,319 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:29:12,319 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:29:12,319 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:29:12,320 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:29:12,320 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:29:12,320 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:29:12,320 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:29:12,320 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:29:12,320 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:29:12,324 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:12,324 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:12,324 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:29:12,324 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:29:12,324 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:29:12,324 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:29:12,324 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:29:12,325 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:12,325 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:12,325 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:29:12,326 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:29:12,326 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@1807823b[Count = 0] remaining members to acquire global barrier 2023-11-19T01:29:12,326 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:29:12,326 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:12,327 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:12,327 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:12,327 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:29:12,327 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:29:12,327 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:29:12,327 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:12,327 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357330074 highest: 1700357330074 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:12,327 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784:(num 1700357330074) roll requested 2023-11-19T01:29:12,339 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357352327, exclude list is [], retry=0 2023-11-19T01:29:12,342 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:29:12,344 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357330074 with entries=15, filesize=4.05 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357352327 2023-11-19T01:29:12,344 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:29:12,344 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357330074 is not closed yet, will try archiving it next time 2023-11-19T01:29:12,344 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta:.meta(num 1700357330092) roll requested 2023-11-19T01:29:12,357 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357352344.meta, exclude list is [], retry=0 2023-11-19T01:29:12,360 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:29:12,362 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357330092.meta with entries=24, filesize=5.77 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357352344.meta 2023-11-19T01:29:12,362 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:29:12,362 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357330092.meta is not closed yet, will try archiving it next time 2023-11-19T01:29:12,367 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-19T01:29:12,368 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357352327 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:12,375 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:29:12,375 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,44997,1700357176784' in zk 2023-11-19T01:29:12,377 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:29:12,377 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:29:12,377 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:29:12,378 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:12,378 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:29:12,378 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:29:12,378 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:29:12,379 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:29:12,379 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:12,379 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:12,380 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:29:12,380 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:12,380 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:12,381 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,44997,1700357176784': 2023-11-19T01:29:12,381 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:29:12,381 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:29:12,381 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:29:12,381 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:29:12,381 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:29:12,381 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:29:12,383 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:29:12,383 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:29:12,383 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:29:12,383 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:29:12,383 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:29:12,383 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:29:12,383 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:29:12,383 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:29:12,384 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:29:12,384 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:12,384 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:29:12,384 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:12,385 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:12,385 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:29:12,385 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:12,386 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:12,390 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:29:12,390 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:29:12,390 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:29:12,390 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:29:12,391 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:29:12,391 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:29:12,391 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:29:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-19T01:29:12,495 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357352494_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } 2023-11-19T01:29:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357352495 (current time:1700357352495). 2023-11-19T01:29:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:29:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357352494_default_test-1700357184679 VERSION not specified, setting to 2 2023-11-19T01:29:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51b5e90d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:12,510 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:37196, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:29:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=155, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1700357352494_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } 2023-11-19T01:29:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357352494_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 }, snapshot procedure id = 155 2023-11-19T01:29:12,515 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1700357352494_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:29:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=155 2023-11-19T01:29:12,516 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1700357352494_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:29:12,518 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357352494_default_test-1700357184679 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:29:12,521 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1700357352494_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:29:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=155 2023-11-19T01:29:12,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=155 2023-11-19T01:29:12,926 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1700357352494_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:29:12,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE; SnapshotRegionProcedure b696d85cf645e95a934c0b80f486a6f6}] 2023-11-19T01:29:12,928 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=156, ppid=155, state=RUNNABLE; SnapshotRegionProcedure b696d85cf645e95a934c0b80f486a6f6 2023-11-19T01:29:12,976 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:29:13,078 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:13,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=156 2023-11-19T01:29:13,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6. 2023-11-19T01:29:13,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.HRegion(2520): Flush status journal for b696d85cf645e95a934c0b80f486a6f6: 2023-11-19T01:29:13,079 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6. for snapshot_1700357352494_default_test-1700357184679 completed. 2023-11-19T01:29:13,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(240): Storing 'test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6.' region-info for snapshot=snapshot_1700357352494_default_test-1700357184679 2023-11-19T01:29:13,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:29:13,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6/f/8cf00b857f704ccba204d7492145d044_SeqId_4_] hfiles 2023-11-19T01:29:13,080 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6/f/8cf00b857f704ccba204d7492145d044_SeqId_4_ for snapshot=snapshot_1700357352494_default_test-1700357184679 2023-11-19T01:29:13,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=155 2023-11-19T01:29:13,485 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6. 2023-11-19T01:29:13,485 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=156}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=156 2023-11-19T01:29:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=156 2023-11-19T01:29:13,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357352494_default_test-1700357184679 on region b696d85cf645e95a934c0b80f486a6f6 2023-11-19T01:29:13,487 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=156, ppid=155, state=RUNNABLE; SnapshotRegionProcedure b696d85cf645e95a934c0b80f486a6f6 2023-11-19T01:29:13,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=156, resume processing ppid=155 2023-11-19T01:29:13,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=156, ppid=155, state=SUCCESS; SnapshotRegionProcedure b696d85cf645e95a934c0b80f486a6f6 in 561 msec 2023-11-19T01:29:13,488 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1700357352494_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:29:13,489 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1700357352494_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:29:13,489 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1700357352494_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:29:13,489 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357352494_default_test-1700357184679 2023-11-19T01:29:13,490 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357352494_default_test-1700357184679 2023-11-19T01:29:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=155 2023-11-19T01:29:13,899 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1700357352494_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:29:13,903 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1700357352494_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:29:13,903 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357352494_default_test-1700357184679 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357352494_default_test-1700357184679 2023-11-19T01:29:13,904 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=155, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1700357352494_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:29:13,904 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357352494_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 }, snapshot procedure id = 155 2023-11-19T01:29:13,905 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=155, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=155, snapshot={ ss=snapshot_1700357352494_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } in 1.3930 sec 2023-11-19T01:29:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=155 2023-11-19T01:29:14,622 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1700357348169 2023-11-19T01:29:14,622 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-19T01:29:14,622 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-19T01:29:14,622 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357348169_test-1700357184679 2023-11-19T01:29:14,622 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357352494_default_test-1700357184679 to hdfs://localhost:42739/backupUT/backup_1700357348169/default/test-1700357184679/ 2023-11-19T01:29:14,622 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:29:14,645 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:29:14,645 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,44889,1700357173572_1372444502_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42739/backupUT/backup_1700357348169/default/test-1700357184679, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42739/backupUT/backup_1700357348169/default/test-1700357184679/.hbase-snapshot/.tmp/snapshot_1700357352494_default_test-1700357184679 2023-11-19T01:29:14,645 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af, snapshotDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357352494_default_test-1700357184679. 2023-11-19T01:29:14,653 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357352494_default_test-1700357184679 to hdfs://localhost:42739/backupUT/backup_1700357348169/default/test-1700357184679/.hbase-snapshot/.tmp/snapshot_1700357352494_default_test-1700357184679 2023-11-19T01:29:14,784 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_ns2_table_table2 2023-11-19T01:29:15,068 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:15,069 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:15,069 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:15,069 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:16,082 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-8387770358158185809.jar 2023-11-19T01:29:16,083 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:16,083 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:16,083 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:16,084 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:16,084 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:16,084 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:16,085 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:16,085 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:16,086 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:16,087 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:29:16,088 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:29:16,088 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:29:16,089 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:29:16,090 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:29:16,090 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:29:16,091 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:29:16,091 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:29:16,092 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:29:16,092 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:29:16,093 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:29:16,093 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:29:16,094 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:16,094 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:16,095 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:16,095 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:16,095 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:16,096 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:16,096 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:16,138 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357352494_default_test-1700357184679' hfile list 2023-11-19T01:29:16,140 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-19T01:29:16,208 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357356174/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:29:16,208 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357356174/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:29:16,258 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-19T01:29:16,281 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6/f/8cf00b857f704ccba204d7492145d044_SeqId_4_ output=hdfs://localhost:42739/backupUT/backup_1700357348169/default/test-1700357184679/archive/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6/f/8cf00b857f704ccba204d7492145d044_SeqId_4_ 2023-11-19T01:29:16,281 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 8.321M/sec 2023-11-19T01:29:17,238 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:29:17,239 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:29:17,244 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357352494_default_test-1700357184679 2023-11-19T01:29:17,244 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357352494_default_test-1700357184679 finished. 2023-11-19T01:29:17,244 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1034): test-1700357184679 2023-11-19T01:29:17,250 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357348169 2023-11-19T01:29:17,250 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:29:17,654 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42739/backupUT/backup_1700357348169/.backup.manifest 2023-11-19T01:29:17,654 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(358): Backup backup_1700357348169 finished: type=FULL,tablelist=test-1700357184679,targetRootDir=hdfs://localhost:42739/backupUT,startts=1700357352306,completets=1700357357249,bytescopied=0 2023-11-19T01:29:17,654 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-19T01:29:17,655 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357352494_default_test-1700357184679 2023-11-19T01:29:17,655 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357352494_default_test-1700357184679" 2023-11-19T01:29:17,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357352494_default_test-1700357184679 2023-11-19T01:29:17,658 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357352494_default_test-1700357184679 for backup backup_1700357348169 succeeded. 2023-11-19T01:29:17,659 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:29:17,662 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:29:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:29:17,664 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:29:17,666 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:29:17,669 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(376): Backup backup_1700357348169 completed. 2023-11-19T01:29:17,669 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:17,679 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestFullRestore(265): backup complete 2023-11-19T01:29:17,679 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-11-19T01:29:17,680 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357348169/.backup.manifest 2023-11-19T01:29:17,681 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357348169/.backup.manifest 2023-11-19T01:29:17,682 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357348169/.backup.manifest 2023-11-19T01:29:17,683 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357348169/.backup.manifest 2023-11-19T01:29:17,683 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(148): Restoring 'faketable' to 'table1' from full backup image hdfs://localhost:42739/backupUT/backup_1700357348169/default/faketable 2023-11-19T01:29:17,700 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleDNE Thread=603 (was 605), OpenFileDescriptor=718 (was 708) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=120 (was 124), ProcessCount=166 (was 166), AvailableMemoryMB=3553 (was 3588) 2023-11-19T01:29:17,700 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=603 is superior to 500 2023-11-19T01:29:17,715 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreSingleCommand Thread=603, OpenFileDescriptor=718, MaxFileDescriptor=60000, SystemLoadAverage=120, ProcessCount=166, AvailableMemoryMB=3552 2023-11-19T01:29:17,715 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=603 is superior to 500 2023-11-19T01:29:17,715 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestFullRestore(76): test full restore on a single table empty table: command-line 2023-11-19T01:29:17,720 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c75b5b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:17,722 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:17,723 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:37212, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:17,724 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:17,726 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:58100, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:29:17,736 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:29:17,740 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:17,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357357740 (current time:1700357357740). 2023-11-19T01:29:17,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:29:17,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:29:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f53f854, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:17,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:17,746 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:37216, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:17,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:17,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:29:17,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=157, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:17,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 157 2023-11-19T01:29:17,750 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:29:17,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=157 2023-11-19T01:29:17,750 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:29:17,752 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:29:17,756 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:29:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=157 2023-11-19T01:29:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=157 2023-11-19T01:29:18,161 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:29:18,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc}] 2023-11-19T01:29:18,163 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=158, ppid=157, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:29:18,314 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:18,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=158 2023-11-19T01:29:18,314 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:29:18,315 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegion(2819): Flushing 322466352cac3ade468ab2ce8e706acc 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-11-19T01:29:18,318 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/e4b8325e01e74ec8a8adfdb749b331fa is 169, key is trslm:hdfs://localhost:42739/backupUT\x00test-1700357184679/meta:log-roll-map/1700357357246/Put/seqid=0 2023-11-19T01:29:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=157 2023-11-19T01:29:18,573 DEBUG [regionserver/jenkins-hbase6:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.6153846153846154, tune throughput to 80.77 MB/second 2023-11-19T01:29:18,746 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/e4b8325e01e74ec8a8adfdb749b331fa 2023-11-19T01:29:18,752 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/178d35e336af41609bd4e366805a6d01 is 452, key is session:backup_1700357348169/session:context/1700357357665/Put/seqid=0 2023-11-19T01:29:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=157 2023-11-19T01:29:19,156 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/178d35e336af41609bd4e366805a6d01 2023-11-19T01:29:19,162 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/e4b8325e01e74ec8a8adfdb749b331fa as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/e4b8325e01e74ec8a8adfdb749b331fa 2023-11-19T01:29:19,167 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/e4b8325e01e74ec8a8adfdb749b331fa, entries=4, sequenceid=127, filesize=5.5 K 2023-11-19T01:29:19,168 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/178d35e336af41609bd4e366805a6d01 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/178d35e336af41609bd4e366805a6d01 2023-11-19T01:29:19,172 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/178d35e336af41609bd4e366805a6d01, entries=2, sequenceid=127, filesize=5.4 K 2023-11-19T01:29:19,172 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 322466352cac3ade468ab2ce8e706acc in 857ms, sequenceid=127, compaction requested=true 2023-11-19T01:29:19,172 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.HRegion(2520): Flush status journal for 322466352cac3ade468ab2ce8e706acc: 2023-11-19T01:29:19,172 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. for snapshot_backup_system completed. 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/83b8d86696ce4c728d410b325dfe4099, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/817bbb8696494171be63b200e480be74, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/298b24d86eff42c285c52df630edc72c, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/e32b4b3110b04f12a891660d01b34d8c, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/a356c57983bf487eaa945a53843b801e, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/10d7714ea1534d16b9a707aa59fac0a0, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5dd4a91274764dc0b74a4aac38eaae51, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/e4b8325e01e74ec8a8adfdb749b331fa] hfiles 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (1/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757 for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (2/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63 for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (3/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2 for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (4/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/83b8d86696ce4c728d410b325dfe4099 for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (5/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/817bbb8696494171be63b200e480be74 for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (6/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/298b24d86eff42c285c52df630edc72c for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (7/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/e32b4b3110b04f12a891660d01b34d8c for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (8/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/a356c57983bf487eaa945a53843b801e for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (9/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/10d7714ea1534d16b9a707aa59fac0a0 for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (10/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5dd4a91274764dc0b74a4aac38eaae51 for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (11/11): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/e4b8325e01e74ec8a8adfdb749b331fa for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/05f6b05b598842a0a5af3e603f2dfa9f, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/6e39103009374e778431dfd7316f5a4c, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/1b2851533774422da3cbc0adf48207b8, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/20bd2650ab284e0b9d4e6d4318baa9b6, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/4b8411bfe401452da4354337a869a577, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/5396f5846ade4d52893b3744b3e7409e, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/dea7ca42cee94103a53d6e0df4cd47bd, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/178d35e336af41609bd4e366805a6d01] hfiles 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (1/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99 for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (2/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (3/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460 for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (4/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1 for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (5/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/05f6b05b598842a0a5af3e603f2dfa9f for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (6/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/6e39103009374e778431dfd7316f5a4c for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (7/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/1b2851533774422da3cbc0adf48207b8 for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (8/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/20bd2650ab284e0b9d4e6d4318baa9b6 for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (9/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/4b8411bfe401452da4354337a869a577 for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (10/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/5396f5846ade4d52893b3744b3e7409e for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (11/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/dea7ca42cee94103a53d6e0df4cd47bd for snapshot=snapshot_backup_system 2023-11-19T01:29:19,173 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] snapshot.SnapshotManifest(264): Adding reference for file (12/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/178d35e336af41609bd4e366805a6d01 for snapshot=snapshot_backup_system 2023-11-19T01:29:19,578 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:29:19,578 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=158}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=158 2023-11-19T01:29:19,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=158 2023-11-19T01:29:19,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:29:19,580 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=158, ppid=157, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:29:19,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=158, resume processing ppid=157 2023-11-19T01:29:19,581 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:29:19,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=158, ppid=157, state=SUCCESS; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc in 1.4190 sec 2023-11-19T01:29:19,582 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:29:19,582 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:29:19,582 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:29:19,583 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:29:19,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=157 2023-11-19T01:29:19,991 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:29:20,004 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:29:20,004 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:29:20,005 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=157, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:29:20,005 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 157 2023-11-19T01:29:20,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=157, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=157, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2580 sec 2023-11-19T01:29:20,286 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:29:21,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=157 2023-11-19T01:29:21,856 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(115): Backup backup_1700357357720 started at 1700357361856. 2023-11-19T01:29:21,858 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(119): Backup session backup_1700357357720 has been started. 2023-11-19T01:29:21,862 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-19T01:29:21,863 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:29:21,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:29:21,863 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:29:21,863 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:29:21,864 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:29:21,864 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:29:21,864 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:29:21,866 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:21,866 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:29:21,866 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:29:21,866 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:29:21,866 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:29:21,867 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:29:21,867 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:29:21,867 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:29:21,867 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:29:21,867 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:29:21,867 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:29:21,867 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:29:21,868 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:29:21,868 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:29:21,868 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:29:21,870 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:21,870 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:21,870 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:29:21,870 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:29:21,871 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:29:21,871 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:29:21,871 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:29:21,871 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:21,871 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:21,872 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:29:21,872 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:29:21,872 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@71e0021b[Count = 0] remaining members to acquire global barrier 2023-11-19T01:29:21,872 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:29:21,872 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:21,877 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:21,877 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:21,877 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:29:21,877 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:29:21,877 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:29:21,878 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:21,878 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357352327 highest: 1700357352327 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:21,878 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784:(num 1700357352327) roll requested 2023-11-19T01:29:21,889 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357361878, exclude list is [], retry=0 2023-11-19T01:29:21,892 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:29:21,894 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357352327 with entries=10, filesize=2.31 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357361878 2023-11-19T01:29:21,894 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:29:21,894 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357352327 is not closed yet, will try archiving it next time 2023-11-19T01:29:21,894 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta:.meta(num 1700357352344) roll requested 2023-11-19T01:29:21,894 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357330074 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/oldWALs/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357330074 2023-11-19T01:29:21,905 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357361894.meta, exclude list is [], retry=0 2023-11-19T01:29:21,908 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:29:21,910 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357352344.meta with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357361894.meta 2023-11-19T01:29:21,910 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:29:21,910 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357352344.meta is not closed yet, will try archiving it next time 2023-11-19T01:29:21,912 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357352344.meta to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/oldWALs/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357352344.meta 2023-11-19T01:29:21,918 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 40 2023-11-19T01:29:21,918 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357361878 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:21,927 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:29:21,927 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,44997,1700357176784' in zk 2023-11-19T01:29:21,929 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:29:21,929 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:21,929 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:29:21,929 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:29:21,929 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:29:21,930 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:29:21,931 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:29:21,931 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:29:21,931 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:21,932 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:21,932 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:29:21,932 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:21,933 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:21,933 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,44997,1700357176784': 2023-11-19T01:29:21,933 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:29:21,933 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:29:21,934 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:29:21,934 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:29:21,934 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:29:21,934 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:29:21,936 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:29:21,936 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:29:21,936 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:29:21,936 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:29:21,936 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:29:21,936 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:29:21,936 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:29:21,936 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:29:21,937 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:21,937 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:29:21,937 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:29:21,937 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:21,938 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:21,938 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:29:21,939 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:21,939 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:21,943 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:29:21,943 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:29:21,943 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:29:21,943 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:29:21,943 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:29:21,944 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:29:21,944 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:29:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-19T01:29:22,047 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357362046_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } 2023-11-19T01:29:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357362047 (current time:1700357362047). 2023-11-19T01:29:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:29:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357362046_default_test-1700357184679 VERSION not specified, setting to 2 2023-11-19T01:29:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1279c4ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:22,055 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:38870, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:29:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=159, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1700357362046_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } 2023-11-19T01:29:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357362046_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 }, snapshot procedure id = 159 2023-11-19T01:29:22,059 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1700357362046_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:29:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=159 2023-11-19T01:29:22,059 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1700357362046_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:29:22,062 WARN [PEWorker-5 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357362046_default_test-1700357184679 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:29:22,064 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1700357362046_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:29:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=159 2023-11-19T01:29:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=159 2023-11-19T01:29:22,470 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1700357362046_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:29:22,470 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE; SnapshotRegionProcedure b696d85cf645e95a934c0b80f486a6f6}] 2023-11-19T01:29:22,472 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=160, ppid=159, state=RUNNABLE; SnapshotRegionProcedure b696d85cf645e95a934c0b80f486a6f6 2023-11-19T01:29:22,622 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:22,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=160 2023-11-19T01:29:22,623 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6. 2023-11-19T01:29:22,623 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.HRegion(2520): Flush status journal for b696d85cf645e95a934c0b80f486a6f6: 2023-11-19T01:29:22,623 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.SnapshotRegionCallable(75): Snapshotting region test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6. for snapshot_1700357362046_default_test-1700357184679 completed. 2023-11-19T01:29:22,623 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(240): Storing 'test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6.' region-info for snapshot=snapshot_1700357362046_default_test-1700357184679 2023-11-19T01:29:22,623 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:29:22,623 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6/f/8cf00b857f704ccba204d7492145d044_SeqId_4_] hfiles 2023-11-19T01:29:22,623 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6/f/8cf00b857f704ccba204d7492145d044_SeqId_4_ for snapshot=snapshot_1700357362046_default_test-1700357184679 2023-11-19T01:29:22,632 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6. 2023-11-19T01:29:22,632 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=160}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=160 2023-11-19T01:29:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=160 2023-11-19T01:29:22,633 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357362046_default_test-1700357184679 on region b696d85cf645e95a934c0b80f486a6f6 2023-11-19T01:29:22,634 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=160, ppid=159, state=RUNNABLE; SnapshotRegionProcedure b696d85cf645e95a934c0b80f486a6f6 2023-11-19T01:29:22,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=160, resume processing ppid=159 2023-11-19T01:29:22,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=160, ppid=159, state=SUCCESS; SnapshotRegionProcedure b696d85cf645e95a934c0b80f486a6f6 in 165 msec 2023-11-19T01:29:22,636 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1700357362046_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:29:22,637 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1700357362046_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:29:22,637 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1700357362046_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:29:22,637 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357362046_default_test-1700357184679 2023-11-19T01:29:22,638 DEBUG [PEWorker-1 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357362046_default_test-1700357184679 2023-11-19T01:29:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=159 2023-11-19T01:29:23,050 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1700357362046_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:29:23,055 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1700357362046_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:29:23,055 DEBUG [PEWorker-1 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357362046_default_test-1700357184679 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357362046_default_test-1700357184679 2023-11-19T01:29:23,056 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=159, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1700357362046_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:29:23,057 DEBUG [PEWorker-1 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357362046_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 }, snapshot procedure id = 159 2023-11-19T01:29:23,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=159, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=159, snapshot={ ss=snapshot_1700357362046_default_test-1700357184679 table=test-1700357184679 type=FLUSH ttl=0 } in 1.0010 sec 2023-11-19T01:29:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=159 2023-11-19T01:29:23,163 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1700357357720 2023-11-19T01:29:23,163 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-19T01:29:23,163 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(80): There are 1 snapshots to be copied. 2023-11-19T01:29:23,163 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357357720_test-1700357184679 2023-11-19T01:29:23,163 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357362046_default_test-1700357184679 to hdfs://localhost:42739/backupUT/backup_1700357357720/default/test-1700357184679/ 2023-11-19T01:29:23,163 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:29:23,195 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:29:23,195 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,44889,1700357173572_1880229250_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42739/backupUT/backup_1700357357720/default/test-1700357184679, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42739/backupUT/backup_1700357357720/default/test-1700357184679/.hbase-snapshot/.tmp/snapshot_1700357362046_default_test-1700357184679 2023-11-19T01:29:23,195 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af, snapshotDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357362046_default_test-1700357184679. 2023-11-19T01:29:23,207 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357362046_default_test-1700357184679 to hdfs://localhost:42739/backupUT/backup_1700357357720/default/test-1700357184679/.hbase-snapshot/.tmp/snapshot_1700357362046_default_test-1700357184679 2023-11-19T01:29:23,636 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:23,636 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:23,637 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:23,637 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:24,641 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-2262554867873260887.jar 2023-11-19T01:29:24,641 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:24,641 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:24,642 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:24,642 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:24,642 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:24,643 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:24,643 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:24,643 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:24,644 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:24,644 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:29:24,645 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:29:24,645 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:29:24,646 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:29:24,646 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:29:24,646 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:29:24,647 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:29:24,647 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:29:24,647 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:29:24,648 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:29:24,648 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:29:24,648 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:29:24,649 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:24,649 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:24,650 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:24,650 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:24,651 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:24,651 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:24,652 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:24,705 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357362046_default_test-1700357184679' hfile list 2023-11-19T01:29:24,707 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-19T01:29:24,778 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357364741/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:29:24,778 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357364741/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:29:24,830 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-19T01:29:24,853 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6/f/8cf00b857f704ccba204d7492145d044_SeqId_4_ output=hdfs://localhost:42739/backupUT/backup_1700357357720/default/test-1700357184679/archive/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6/f/8cf00b857f704ccba204d7492145d044_SeqId_4_ 2023-11-19T01:29:24,853 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8725 (8.5 K) time=0sec 8.321M/sec 2023-11-19T01:29:25,809 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:29:25,810 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:29:25,815 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357362046_default_test-1700357184679 2023-11-19T01:29:25,815 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357362046_default_test-1700357184679 finished. 2023-11-19T01:29:25,815 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1034): test-1700357184679 2023-11-19T01:29:25,820 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357357720 2023-11-19T01:29:25,820 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:29:26,225 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42739/backupUT/backup_1700357357720/.backup.manifest 2023-11-19T01:29:26,225 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(358): Backup backup_1700357357720 finished: type=FULL,tablelist=test-1700357184679,targetRootDir=hdfs://localhost:42739/backupUT,startts=1700357361856,completets=1700357365820,bytescopied=0 2023-11-19T01:29:26,225 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-19T01:29:26,225 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357362046_default_test-1700357184679 2023-11-19T01:29:26,226 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357362046_default_test-1700357184679" 2023-11-19T01:29:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357362046_default_test-1700357184679 2023-11-19T01:29:26,229 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357362046_default_test-1700357184679 for backup backup_1700357357720 succeeded. 2023-11-19T01:29:26,229 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:29:26,233 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:29:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:29:26,235 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:29:26,236 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:29:26,241 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(376): Backup backup_1700357357720 completed. 2023-11-19T01:29:26,241 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:26,241 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestFullRestore(80): backup complete 2023-11-19T01:29:26,256 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5331ddc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:26,257 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:26,259 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:38884, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:26,259 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(101): HBase table table1 does not exist. It will be created during restore process 2023-11-19T01:29:26,259 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357357720/.backup.manifest 2023-11-19T01:29:26,261 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357357720/.backup.manifest 2023-11-19T01:29:26,262 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357357720/.backup.manifest 2023-11-19T01:29:26,263 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357357720/.backup.manifest 2023-11-19T01:29:26,263 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(148): Restoring 'test-1700357184679' to 'table1' from full backup image hdfs://localhost:42739/backupUT/backup_1700357357720/default/test-1700357184679 2023-11-19T01:29:26,269 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(487): Creating target table 'table1' 2023-11-19T01:29:26,269 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(391): Parsing region dir: hdfs://localhost:42739/backupUT/backup_1700357357720/default/test-1700357184679/archive/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6 2023-11-19T01:29:26,270 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(421): Parsing family dir [hdfs://localhost:42739/backupUT/backup_1700357357720/default/test-1700357184679/archive/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6/f in region [hdfs://localhost:42739/backupUT/backup_1700357357720/default/test-1700357184679/archive/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6] 2023-11-19T01:29:26,272 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.RestoreTool(442): Trying to figure out region boundaries hfile=hdfs://localhost:42739/backupUT/backup_1700357357720/default/test-1700357184679/archive/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6/f/8cf00b857f704ccba204d7492145d044_SeqId_4_ first=row0 last=row98 2023-11-19T01:29:26,273 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:26,274 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:39618, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:29:26,275 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$4(2333): Client=jenkins//172.31.2.20 create 'table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2023-11-19T01:29:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=161, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=table1 2023-11-19T01:29:26,277 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2023-11-19T01:29:26,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(712): Client=jenkins//172.31.2.20 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 161 2023-11-19T01:29:26,278 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2023-11-19T01:29:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=161 2023-11-19T01:29:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=161 2023-11-19T01:29:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=161 2023-11-19T01:29:26,685 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7075): creating {ENCODED => e44492b3c4cb4be05bdade515e7a5c8b, NAME => 'table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'f', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:29:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=161 2023-11-19T01:29:27,091 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(888): Instantiated table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:27,091 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1668): Closing e44492b3c4cb4be05bdade515e7a5c8b, disabling compactions & flushes 2023-11-19T01:29:27,091 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1690): Closing region table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b. 2023-11-19T01:29:27,091 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b. 2023-11-19T01:29:27,091 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1778): Acquired close lock on table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b. after waiting 0 ms 2023-11-19T01:29:27,091 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1788): Updates disabled for region table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b. 2023-11-19T01:29:27,091 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1909): Closed table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b. 2023-11-19T01:29:27,092 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1622): Region close journal for e44492b3c4cb4be05bdade515e7a5c8b: 2023-11-19T01:29:27,093 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2023-11-19T01:29:27,093 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1700357367093"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1700357367093"}]},"ts":"1700357367093"} 2023-11-19T01:29:27,094 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2023-11-19T01:29:27,095 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2023-11-19T01:29:27,095 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357367095"}]},"ts":"1700357367095"} 2023-11-19T01:29:27,096 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLING in hbase:meta 2023-11-19T01:29:27,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=e44492b3c4cb4be05bdade515e7a5c8b, ASSIGN}] 2023-11-19T01:29:27,103 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=table1, region=e44492b3c4cb4be05bdade515e7a5c8b, ASSIGN 2023-11-19T01:29:27,103 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(262): Starting pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=table1, region=e44492b3c4cb4be05bdade515e7a5c8b, ASSIGN; state=OFFLINE, location=jenkins-hbase6.apache.org,44997,1700357176784; forceNewPlan=false, retain=false 2023-11-19T01:29:27,256 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=162 updating hbase:meta row=e44492b3c4cb4be05bdade515e7a5c8b, regionState=OPENING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:27,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE; OpenRegionProcedure e44492b3c4cb4be05bdade515e7a5c8b, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:29:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=161 2023-11-19T01:29:27,411 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:27,414 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] handler.AssignRegionHandler(130): Open table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b. 2023-11-19T01:29:27,414 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7254): Opening region: {ENCODED => e44492b3c4cb4be05bdade515e7a5c8b, NAME => 'table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b.', STARTKEY => '', ENDKEY => ''} 2023-11-19T01:29:27,415 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] coprocessor.CoprocessorHost(174): System coprocessor org.apache.hadoop.hbase.backup.BackupObserver loaded, priority=536870911. 2023-11-19T01:29:27,415 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 e44492b3c4cb4be05bdade515e7a5c8b 2023-11-19T01:29:27,415 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(888): Instantiated table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2023-11-19T01:29:27,415 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7296): checking encryption for e44492b3c4cb4be05bdade515e7a5c8b 2023-11-19T01:29:27,415 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(7299): checking classloading for e44492b3c4cb4be05bdade515e7a5c8b 2023-11-19T01:29:27,416 INFO [StoreOpener-e44492b3c4cb4be05bdade515e7a5c8b-1 {}] regionserver.HStore(397): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family f of region e44492b3c4cb4be05bdade515e7a5c8b 2023-11-19T01:29:27,417 INFO [StoreOpener-e44492b3c4cb4be05bdade515e7a5c8b-1 {}] compactions.CompactionConfiguration(173): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e44492b3c4cb4be05bdade515e7a5c8b columnFamilyName f 2023-11-19T01:29:27,417 DEBUG [StoreOpener-e44492b3c4cb4be05bdade515e7a5c8b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2023-11-19T01:29:27,418 INFO [StoreOpener-e44492b3c4cb4be05bdade515e7a5c8b-1 {}] regionserver.HStore(324): Store=e44492b3c4cb4be05bdade515e7a5c8b/f, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2023-11-19T01:29:27,418 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/e44492b3c4cb4be05bdade515e7a5c8b 2023-11-19T01:29:27,418 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(5270): Found 0 recovered edits file(s) under hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/e44492b3c4cb4be05bdade515e7a5c8b 2023-11-19T01:29:27,420 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(1076): writing seq id for e44492b3c4cb4be05bdade515e7a5c8b 2023-11-19T01:29:27,421 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/e44492b3c4cb4be05bdade515e7a5c8b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2023-11-19T01:29:27,422 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(1093): Opened e44492b3c4cb4be05bdade515e7a5c8b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=9406983840, jitterRate=-0.12390635907649994}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2023-11-19T01:29:27,422 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegion(992): Region open journal for e44492b3c4cb4be05bdade515e7a5c8b: 2023-11-19T01:29:27,423 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegionServer(2556): Post open deploy tasks for table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b., pid=163, masterSystemTime=1700357367411 2023-11-19T01:29:27,424 DEBUG [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] regionserver.HRegionServer(2583): Finished post open deploy task for table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b. 2023-11-19T01:29:27,424 INFO [RS_OPEN_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_OPEN_REGION, pid=163}] handler.AssignRegionHandler(158): Opened table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b. 2023-11-19T01:29:27,424 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=162 updating hbase:meta row=e44492b3c4cb4be05bdade515e7a5c8b, regionState=OPEN, openSeqNum=2, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:27,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=163, resume processing ppid=162 2023-11-19T01:29:27,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=163, ppid=162, state=SUCCESS; OpenRegionProcedure e44492b3c4cb4be05bdade515e7a5c8b, server=jenkins-hbase6.apache.org,44997,1700357176784 in 167 msec 2023-11-19T01:29:27,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=162, resume processing ppid=161 2023-11-19T01:29:27,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=162, ppid=161, state=SUCCESS; TransitRegionStateProcedure table=table1, region=e44492b3c4cb4be05bdade515e7a5c8b, ASSIGN in 326 msec 2023-11-19T01:29:27,428 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2023-11-19T01:29:27,429 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357367428"}]},"ts":"1700357367428"} 2023-11-19T01:29:27,429 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=ENABLED in hbase:meta 2023-11-19T01:29:27,432 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(81): pid=161, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2023-11-19T01:29:27,432 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=161, state=SUCCESS; CreateTableProcedure table=table1 in 1.1570 sec 2023-11-19T01:29:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=161 2023-11-19T01:29:28,388 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(64): Restore full backup from directory hdfs://localhost:42739/backupUT/backup_1700357357720/default/test-1700357184679/archive/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6 from hbase tables test-1700357184679 to tables table1 2023-11-19T01:29:28,388 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(72): Restore test-1700357184679 into table1 2023-11-19T01:29:28,390 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceHFileSplitterJob(107): add incremental job :hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-table1-1700357368388 from hdfs://localhost:42739/backupUT/backup_1700357357720/default/test-1700357184679/archive/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6 2023-11-19T01:29:28,396 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@792bb412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:28,397 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:28,398 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:39632, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:29:28,399 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(672): bulkload locality sensitive enabled 2023-11-19T01:29:28,399 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(535): Looking up current regions for table table1 2023-11-19T01:29:28,402 DEBUG [hconnection-0x69a1a3fe-shared-pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:28,403 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:38890, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:28,404 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(544): SplitPoint startkey for table1: 2023-11-19T01:29:28,404 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(694): Configuring 1 reduce partitions to match current region count for all tables 2023-11-19T01:29:28,404 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(559): Writing partition information to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/partitions_04d806a8-3f02-42e4-8d6f-0b1a2086db7b 2023-11-19T01:29:28,810 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:28,810 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:28,811 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:28,811 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:29,815 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-615015242588825708.jar 2023-11-19T01:29:29,815 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:29,815 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:29,816 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:29,816 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:29,816 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:29,817 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:29,817 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:29,817 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:29,818 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:29,818 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:29:29,819 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:29:29,819 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:29:29,820 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:29:29,821 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:29:29,821 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:29:29,822 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:29:29,822 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:29:29,823 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:29:29,824 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:29:29,825 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:29:29,825 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:29:29,826 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:29,826 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:29,827 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:29,827 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.io.ImmutableBytesWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:29,828 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.util.MapReduceExtendedCell, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:29,828 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.HFileOutputFormat2, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:29,830 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:29,830 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.HFileOutputFormat2(714): Incremental output configured for tables: table1 2023-11-19T01:29:29,830 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:29,830 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceHFileSplitterJob(119): success configuring load incremental job 2023-11-19T01:29:29,830 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(796): The addDependencyJars(Configuration, Class...) method has been deprecated since it is easy to use incorrectly. Most users should rely on addDependencyJars(Job) instead. See HBASE-8386 for more details. 2023-11-19T01:29:29,831 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.base.Preconditions, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:29:29,843 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.JobResourceUploader(470): No job jar file set. User classes may not be found. See Job or Job#setJar(String). 2023-11-19T01:29:29,917 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357369884/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:29:29,917 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357369884/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:29:29,963 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(88): Initialize HFileRecordReader for hdfs://localhost:42739/backupUT/backup_1700357357720/default/test-1700357184679/archive/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6/f/8cf00b857f704ccba204d7492145d044_SeqId_4_ 2023-11-19T01:29:29,967 INFO [LocalJobRunner Map Task Executor #0 {}] mapreduce.HFileInputFormat$HFileRecordReader(101): Seeking to start 2023-11-19T01:29:29,985 DEBUG [pool-536-thread-1 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d4d956, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:29,986 DEBUG [hconnection-0x7810dc0f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:29,988 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:38894, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:29,991 DEBUG [pool-536-thread-1 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:29,991 DEBUG [pool-536-thread-1 {}] mapreduce.HFileOutputFormat2$1(339): First rowkey: [row0] 2023-11-19T01:29:29,991 DEBUG [pool-536-thread-1 {}] mapreduce.HFileOutputFormat2$1(346): Use favored nodes writer: jenkins-hbase6.apache.org 2023-11-19T01:29:29,996 DEBUG [pool-536-thread-1 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-table1-1700357368388/_temporary/0/_temporary/attempt_local447782707_0025_r_000000_0/f/cedec2ba5d94447f9c2e11001a8e5015 is 37, key is row10/f:q1/1700357187320/Put/seqid=0 2023-11-19T01:29:29,997 WARN [Thread-2351 {}] hdfs.DataStreamer(1828): These favored nodes were specified but not chosen: [jenkins-hbase6.apache.org:44997] Specified favored nodes: [jenkins-hbase6.apache.org:44997] 2023-11-19T01:29:30,960 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(89): Restoring HFiles from directory hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-table1-1700357368388 2023-11-19T01:29:30,966 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b910f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:30,967 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:30,968 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:40088, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:30,973 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] tool.LoadIncrementalHFiles(1067): Skipping non-directory hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-table1-1700357368388/_SUCCESS 2023-11-19T01:29:30,976 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:30,978 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:52000, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:29:30,985 INFO [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles(794): Trying to load hfile=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-table1-1700357368388/f/cedec2ba5d94447f9c2e11001a8e5015 first=Optional[row0] last=Optional[row98] 2023-11-19T01:29:30,986 DEBUG [LoadIncrementalHFiles-0 {}] tool.LoadIncrementalHFiles$1(559): Going to connect to server region=table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b., hostname=jenkins-hbase6.apache.org,44997,1700357176784, seqNum=2 for row with hfile group [{f,hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-table1-1700357368388/f/cedec2ba5d94447f9c2e11001a8e5015}] 2023-11-19T01:29:30,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(611): Validating hfile at hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-table1-1700357368388/f/cedec2ba5d94447f9c2e11001a8e5015 for inclusion in e44492b3c4cb4be05bdade515e7a5c8b/f 2023-11-19T01:29:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(623): HFile bounds: first=row0 last=row98 2023-11-19T01:29:30,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(625): Region bounds: first= last= 2023-11-19T01:29:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegion(2520): Flush status journal for e44492b3c4cb4be05bdade515e7a5c8b: 2023-11-19T01:29:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(397): Moving hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/hbase-staging/bulk_output-default-table1-1700357368388/f/cedec2ba5d94447f9c2e11001a8e5015 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__table1__cge6h6ffnle9bd2bqsgtf2laqljl47adn58rq83njoikulsqb2ptk7e1spqtv38a/f/cedec2ba5d94447f9c2e11001a8e5015 2023-11-19T01:29:30,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__table1__cge6h6ffnle9bd2bqsgtf2laqljl47adn58rq83njoikulsqb2ptk7e1spqtv38a/f/cedec2ba5d94447f9c2e11001a8e5015 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/e44492b3c4cb4be05bdade515e7a5c8b/f/88544e86e772408eb7327cdd055af3dd_SeqId_4_ 2023-11-19T01:29:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a1653e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:31,004 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:52006, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:29:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:31,008 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:40104, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:31,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(697): Loaded HFile hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__table1__cge6h6ffnle9bd2bqsgtf2laqljl47adn58rq83njoikulsqb2ptk7e1spqtv38a/f/cedec2ba5d94447f9c2e11001a8e5015 into e44492b3c4cb4be05bdade515e7a5c8b/f as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/e44492b3c4cb4be05bdade515e7a5c8b/f/88544e86e772408eb7327cdd055af3dd_SeqId_4_ - updating store file list. 2023-11-19T01:29:31,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(719): Loaded HFile hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/e44492b3c4cb4be05bdade515e7a5c8b/f/88544e86e772408eb7327cdd055af3dd_SeqId_4_ into e44492b3c4cb4be05bdade515e7a5c8b/f 2023-11-19T01:29:31,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.HStore(703): Successfully loaded hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__table1__cge6h6ffnle9bd2bqsgtf2laqljl47adn58rq83njoikulsqb2ptk7e1spqtv38a/f/cedec2ba5d94447f9c2e11001a8e5015 into e44492b3c4cb4be05bdade515e7a5c8b/f (new location: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/e44492b3c4cb4be05bdade515e7a5c8b/f/88544e86e772408eb7327cdd055af3dd_SeqId_4_) 2023-11-19T01:29:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] regionserver.SecureBulkLoadManager$SecureBulkLoadListener(412): Bulk Load done for: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/staging/jenkins__table1__cge6h6ffnle9bd2bqsgtf2laqljl47adn58rq83njoikulsqb2ptk7e1spqtv38a/f/cedec2ba5d94447f9c2e11001a8e5015 2023-11-19T01:29:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d5e1aa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:31,033 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:52008, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=MasterService 2023-11-19T01:29:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:31,037 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:40106, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2023-11-19T01:29:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44997 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:31,050 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:31,050 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceRestoreJob(100): Restore Job finished:0 2023-11-19T01:29:31,050 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(231): Restore includes the following image(s): 2023-11-19T01:29:31,050 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(233): Backup: backup_1700357357720 hdfs://localhost:42739/backupUT/backup_1700357357720/default/test-1700357184679/ 2023-11-19T01:29:31,050 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(238): restoreStage finished 2023-11-19T01:29:31,050 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:31,052 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$13(2695): Client=jenkins//172.31.2.20 disable table1 2023-11-19T01:29:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=table1 2023-11-19T01:29:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=164 2023-11-19T01:29:31,058 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357371058"}]},"ts":"1700357371058"} 2023-11-19T01:29:31,059 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLING in hbase:meta 2023-11-19T01:29:31,061 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(293): Set table1 to state=DISABLING 2023-11-19T01:29:31,062 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=e44492b3c4cb4be05bdade515e7a5c8b, UNASSIGN}] 2023-11-19T01:29:31,063 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=165, ppid=164, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=table1, region=e44492b3c4cb4be05bdade515e7a5c8b, UNASSIGN 2023-11-19T01:29:31,064 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=165 updating hbase:meta row=e44492b3c4cb4be05bdade515e7a5c8b, regionState=CLOSING, regionLocation=jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:31,065 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE; CloseRegionProcedure e44492b3c4cb4be05bdade515e7a5c8b, server=jenkins-hbase6.apache.org,44997,1700357176784}] 2023-11-19T01:29:31,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=164 2023-11-19T01:29:31,216 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:31,216 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] handler.UnassignRegionHandler(119): Close e44492b3c4cb4be05bdade515e7a5c8b 2023-11-19T01:29:31,216 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1668): Closing e44492b3c4cb4be05bdade515e7a5c8b, disabling compactions & flushes 2023-11-19T01:29:31,216 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1690): Closing region table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b. 2023-11-19T01:29:31,216 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1711): Waiting without time limit for close lock on table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b. 2023-11-19T01:29:31,216 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1778): Acquired close lock on table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b. after waiting 0 ms 2023-11-19T01:29:31,216 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1788): Updates disabled for region table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b. 2023-11-19T01:29:31,222 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/e44492b3c4cb4be05bdade515e7a5c8b/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-11-19T01:29:31,222 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:29:31,222 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1909): Closed table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b. 2023-11-19T01:29:31,222 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] regionserver.HRegion(1622): Region close journal for e44492b3c4cb4be05bdade515e7a5c8b: 2023-11-19T01:29:31,224 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=165 updating hbase:meta row=e44492b3c4cb4be05bdade515e7a5c8b, regionState=CLOSED 2023-11-19T01:29:31,225 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION, pid=166}] handler.UnassignRegionHandler(163): Closed e44492b3c4cb4be05bdade515e7a5c8b 2023-11-19T01:29:31,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=166, resume processing ppid=165 2023-11-19T01:29:31,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=166, ppid=165, state=SUCCESS; CloseRegionProcedure e44492b3c4cb4be05bdade515e7a5c8b, server=jenkins-hbase6.apache.org,44997,1700357176784 in 160 msec 2023-11-19T01:29:31,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=165, resume processing ppid=164 2023-11-19T01:29:31,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=165, ppid=164, state=SUCCESS; TransitRegionStateProcedure table=table1, region=e44492b3c4cb4be05bdade515e7a5c8b, UNASSIGN in 164 msec 2023-11-19T01:29:31,228 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1700357371228"}]},"ts":"1700357371228"} 2023-11-19T01:29:31,229 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=table1, state=DISABLED in hbase:meta 2023-11-19T01:29:31,230 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(305): Set table1 to state=DISABLED 2023-11-19T01:29:31,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=164, state=SUCCESS; DisableTableProcedure table=table1 in 179 msec 2023-11-19T01:29:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=164 2023-11-19T01:29:31,360 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster$5(2449): Client=jenkins//172.31.2.20 delete table1 2023-11-19T01:29:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=167, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=table1 2023-11-19T01:29:31,362 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(98): Waiting for RIT for pid=167, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-11-19T01:29:31,363 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(110): Deleting regions from filesystem for pid=167, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=table1 2023-11-19T01:29:31,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=167 2023-11-19T01:29:31,364 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/e44492b3c4cb4be05bdade515e7a5c8b 2023-11-19T01:29:31,365 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/e44492b3c4cb4be05bdade515e7a5c8b/f, FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/e44492b3c4cb4be05bdade515e7a5c8b/recovered.edits] 2023-11-19T01:29:31,369 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/e44492b3c4cb4be05bdade515e7a5c8b/f/88544e86e772408eb7327cdd055af3dd_SeqId_4_ to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/archive/data/default/table1/e44492b3c4cb4be05bdade515e7a5c8b/f/88544e86e772408eb7327cdd055af3dd_SeqId_4_ 2023-11-19T01:29:31,371 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/e44492b3c4cb4be05bdade515e7a5c8b/recovered.edits/6.seqid to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/archive/data/default/table1/e44492b3c4cb4be05bdade515e7a5c8b/recovered.edits/6.seqid 2023-11-19T01:29:31,372 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/table1/e44492b3c4cb4be05bdade515e7a5c8b 2023-11-19T01:29:31,372 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(293): Archived table1 regions 2023-11-19T01:29:31,373 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from META for pid=167, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=table1 2023-11-19T01:29:31,375 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(350): Deleting some vestigial 1 rows of table1 from hbase:meta 2023-11-19T01:29:31,376 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(387): Removing 'table1' descriptor. 2023-11-19T01:29:31,377 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(121): Deleting assignment state for pid=167, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=table1 2023-11-19T01:29:31,377 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(377): Removing 'table1' from region states. 2023-11-19T01:29:31,377 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1700357371377"}]},"ts":"9223372036854775807"} 2023-11-19T01:29:31,378 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2023-11-19T01:29:31,378 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => e44492b3c4cb4be05bdade515e7a5c8b, NAME => 'table1,,1700357366275.e44492b3c4cb4be05bdade515e7a5c8b.', STARTKEY => '', ENDKEY => ''}] 2023-11-19T01:29:31,378 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(381): Marking 'table1' as deleted. 2023-11-19T01:29:31,378 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1700357371378"}]},"ts":"9223372036854775807"} 2023-11-19T01:29:31,379 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table table1 state from META 2023-11-19T01:29:31,382 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(127): Finished pid=167, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=table1 2023-11-19T01:29:31,383 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1413): Finished pid=167, state=SUCCESS; DeleteTableProcedure table=table1 in 22 msec 2023-11-19T01:29:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=167 2023-11-19T01:29:31,480 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreSingleCommand Thread=605 (was 603) - Thread LEAK? -, OpenFileDescriptor=713 (was 718), MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=128 (was 120) - SystemLoadAverage LEAK? -, ProcessCount=166 (was 166), AvailableMemoryMB=3436 (was 3552) 2023-11-19T01:29:31,481 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-11-19T01:29:31,496 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(147): before: backup.TestFullRestore#testFullRestoreMultipleDNE Thread=605, OpenFileDescriptor=713, MaxFileDescriptor=60000, SystemLoadAverage=128, ProcessCount=166, AvailableMemoryMB=3436 2023-11-19T01:29:31,496 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=605 is superior to 500 2023-11-19T01:29:31,496 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] backup.TestFullRestore(302): test restore fails on multiple tables that do not exist 2023-11-19T01:29:31,502 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ea17c84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:31,505 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:31,507 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:40108, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:31,508 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=MasterService, sasl=false 2023-11-19T01:29:31,510 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:52020, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2023-11-19T01:29:31,518 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(630): Start new backup exclusive operation 2023-11-19T01:29:31,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357371523 (current time:1700357371523). 2023-11-19T01:29:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:29:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_backup_system VERSION not specified, setting to 2 2023-11-19T01:29:31,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1782bc67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:31,531 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:40110, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:29:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=168, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } 2023-11-19T01:29:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 168 2023-11-19T01:29:31,534 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:29:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=168 2023-11-19T01:29:31,535 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:29:31,537 WARN [PEWorker-1 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:29:31,540 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:29:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=168 2023-11-19T01:29:31,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=168 2023-11-19T01:29:31,945 INFO [PEWorker-1 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:29:31,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc}] 2023-11-19T01:29:31,947 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=169, ppid=168, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:29:32,097 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:32,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=169 2023-11-19T01:29:32,098 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:29:32,098 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegion(2819): Flushing 322466352cac3ade468ab2ce8e706acc 2/2 column families, dataSize=1.14 KB heapSize=2.22 KB 2023-11-19T01:29:32,102 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/4ad5490ebe7b498aa111f03b6944456d is 169, key is trslm:hdfs://localhost:42739/backupUT\x00test-1700357184679/meta:log-roll-map/1700357365817/Put/seqid=0 2023-11-19T01:29:32,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=168 2023-11-19T01:29:32,506 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=451 B at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/4ad5490ebe7b498aa111f03b6944456d 2023-11-19T01:29:32,512 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/9c800e2266a84f0a8ad747cde82b198a is 452, key is session:backup_1700357357720/session:context/1700357366235/Put/seqid=0 2023-11-19T01:29:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=168 2023-11-19T01:29:32,916 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=721 B at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/9c800e2266a84f0a8ad747cde82b198a 2023-11-19T01:29:32,921 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/4ad5490ebe7b498aa111f03b6944456d as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/4ad5490ebe7b498aa111f03b6944456d 2023-11-19T01:29:32,925 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/4ad5490ebe7b498aa111f03b6944456d, entries=4, sequenceid=138, filesize=5.5 K 2023-11-19T01:29:32,926 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/9c800e2266a84f0a8ad747cde82b198a as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/9c800e2266a84f0a8ad747cde82b198a 2023-11-19T01:29:32,930 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/9c800e2266a84f0a8ad747cde82b198a, entries=2, sequenceid=138, filesize=5.4 K 2023-11-19T01:29:32,931 INFO [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegion(3022): Finished flush of dataSize ~1.14 KB/1172, heapSize ~2.19 KB/2240, currentSize=0 B/0 for 322466352cac3ade468ab2ce8e706acc in 833ms, sequenceid=138, compaction requested=true 2023-11-19T01:29:32,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.HRegion(2520): Flush status journal for 322466352cac3ade468ab2ce8e706acc: 2023-11-19T01:29:32,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.SnapshotRegionCallable(75): Snapshotting region backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. for snapshot_backup_system completed. 2023-11-19T01:29:32,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(240): Storing 'backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc.' region-info for snapshot=snapshot_backup_system 2023-11-19T01:29:32,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:29:32,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/83b8d86696ce4c728d410b325dfe4099, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/817bbb8696494171be63b200e480be74, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/298b24d86eff42c285c52df630edc72c, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/e32b4b3110b04f12a891660d01b34d8c, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/a356c57983bf487eaa945a53843b801e, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/10d7714ea1534d16b9a707aa59fac0a0, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5dd4a91274764dc0b74a4aac38eaae51, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/e4b8325e01e74ec8a8adfdb749b331fa, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/4ad5490ebe7b498aa111f03b6944456d] hfiles 2023-11-19T01:29:32,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (1/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5c700b4d765d4c26bf3c25f07d229757 for snapshot=snapshot_backup_system 2023-11-19T01:29:32,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (2/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/33514f8ae1f64952ad0b4b83073e2d63 for snapshot=snapshot_backup_system 2023-11-19T01:29:32,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (3/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/2523325fca554399aaad3206b6dd67a2 for snapshot=snapshot_backup_system 2023-11-19T01:29:32,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (4/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/83b8d86696ce4c728d410b325dfe4099 for snapshot=snapshot_backup_system 2023-11-19T01:29:32,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (5/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/817bbb8696494171be63b200e480be74 for snapshot=snapshot_backup_system 2023-11-19T01:29:32,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (6/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/298b24d86eff42c285c52df630edc72c for snapshot=snapshot_backup_system 2023-11-19T01:29:32,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (7/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/e32b4b3110b04f12a891660d01b34d8c for snapshot=snapshot_backup_system 2023-11-19T01:29:32,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (8/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/a356c57983bf487eaa945a53843b801e for snapshot=snapshot_backup_system 2023-11-19T01:29:32,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (9/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/10d7714ea1534d16b9a707aa59fac0a0 for snapshot=snapshot_backup_system 2023-11-19T01:29:32,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (10/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/5dd4a91274764dc0b74a4aac38eaae51 for snapshot=snapshot_backup_system 2023-11-19T01:29:32,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (11/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/e4b8325e01e74ec8a8adfdb749b331fa for snapshot=snapshot_backup_system 2023-11-19T01:29:32,931 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (12/12): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/4ad5490ebe7b498aa111f03b6944456d for snapshot=snapshot_backup_system 2023-11-19T01:29:32,932 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/05f6b05b598842a0a5af3e603f2dfa9f, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/6e39103009374e778431dfd7316f5a4c, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/1b2851533774422da3cbc0adf48207b8, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/20bd2650ab284e0b9d4e6d4318baa9b6, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/4b8411bfe401452da4354337a869a577, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/5396f5846ade4d52893b3744b3e7409e, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/dea7ca42cee94103a53d6e0df4cd47bd, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/178d35e336af41609bd4e366805a6d01, hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/9c800e2266a84f0a8ad747cde82b198a] hfiles 2023-11-19T01:29:32,932 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (1/13): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/8670d77389e247fc9026582c1b208b99 for snapshot=snapshot_backup_system 2023-11-19T01:29:32,932 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (2/13): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/fde2616a711d412499f12db80d1c34ff for snapshot=snapshot_backup_system 2023-11-19T01:29:32,932 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (3/13): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b83bc5cc85424cf8b1d3e429f8b40460 for snapshot=snapshot_backup_system 2023-11-19T01:29:32,932 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (4/13): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/b035097b152048f4aa18d7b48c5588c1 for snapshot=snapshot_backup_system 2023-11-19T01:29:32,932 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (5/13): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/05f6b05b598842a0a5af3e603f2dfa9f for snapshot=snapshot_backup_system 2023-11-19T01:29:32,932 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (6/13): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/6e39103009374e778431dfd7316f5a4c for snapshot=snapshot_backup_system 2023-11-19T01:29:32,932 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (7/13): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/1b2851533774422da3cbc0adf48207b8 for snapshot=snapshot_backup_system 2023-11-19T01:29:32,932 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (8/13): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/20bd2650ab284e0b9d4e6d4318baa9b6 for snapshot=snapshot_backup_system 2023-11-19T01:29:32,932 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (9/13): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/4b8411bfe401452da4354337a869a577 for snapshot=snapshot_backup_system 2023-11-19T01:29:32,932 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (10/13): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/5396f5846ade4d52893b3744b3e7409e for snapshot=snapshot_backup_system 2023-11-19T01:29:32,932 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (11/13): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/dea7ca42cee94103a53d6e0df4cd47bd for snapshot=snapshot_backup_system 2023-11-19T01:29:32,932 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (12/13): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/178d35e336af41609bd4e366805a6d01 for snapshot=snapshot_backup_system 2023-11-19T01:29:32,932 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] snapshot.SnapshotManifest(264): Adding reference for file (13/13): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/9c800e2266a84f0a8ad747cde82b198a for snapshot=snapshot_backup_system 2023-11-19T01:29:32,952 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(128): Cannot locate configuration: tried hadoop-metrics2-jobhistoryserver.properties,hadoop-metrics2.properties 2023-11-19T01:29:33,336 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:29:33,336 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-2 {event_type=RS_SNAPSHOT_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=169 2023-11-19T01:29:33,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=169 2023-11-19T01:29:33,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_backup_system on region 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:29:33,338 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=169, ppid=168, state=RUNNABLE; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:29:33,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=169, resume processing ppid=168 2023-11-19T01:29:33,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=169, ppid=168, state=SUCCESS; SnapshotRegionProcedure 322466352cac3ade468ab2ce8e706acc in 1.3940 sec 2023-11-19T01:29:33,340 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:29:33,340 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:29:33,341 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:29:33,341 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_backup_system 2023-11-19T01:29:33,342 DEBUG [PEWorker-4 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system 2023-11-19T01:29:33,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=168 2023-11-19T01:29:33,753 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:29:33,772 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:29:33,772 DEBUG [PEWorker-4 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_backup_system to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_backup_system 2023-11-19T01:29:33,773 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=168, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:29:33,773 DEBUG [PEWorker-4 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 }, snapshot procedure id = 168 2023-11-19T01:29:33,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1413): Finished pid=168, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=168, snapshot={ ss=snapshot_backup_system table=backup:system type=FLUSH ttl=0 } in 2.2410 sec 2023-11-19T01:29:35,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=168 2023-11-19T01:29:35,640 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(115): Backup backup_1700357371502 started at 1700357375640. 2023-11-19T01:29:35,641 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(119): Backup session backup_1700357371502 has been started. 2023-11-19T01:29:35,645 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(150): Execute roll log procedure for full backup ... 2023-11-19T01:29:35,646 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(962): Client=jenkins//172.31.2.20 procedure request for: rolllog-proc 2023-11-19T01:29:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.ProcedureCoordinator(143): Procedure rolllog was in running list but was completed. Accepting new attempt. 2023-11-19T01:29:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.ProcedureCoordinator(165): Submitting procedure rolllog 2023-11-19T01:29:35,646 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(191): Starting procedure 'rolllog' 2023-11-19T01:29:35,646 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 180000 ms 2023-11-19T01:29:35,647 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(199): Procedure 'rolllog' starting 'acquire' 2023-11-19T01:29:35,647 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(241): Starting procedure 'rolllog', kicking off acquire phase on members. 2023-11-19T01:29:35,647 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(92): Creating acquire znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:29:35,649 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(100): Watching for acquire node:/1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:35,649 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:29:35,649 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:29:35,649 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(203): Waiting for all members to 'acquire' 2023-11-19T01:29:35,649 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(186): Found procedure znode: /1/rolllog-proc/acquired/rolllog 2023-11-19T01:29:35,650 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(212): start proc data length is 35 2023-11-19T01:29:35,650 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(214): Found data for znode:/1/rolllog-proc/acquired/rolllog 2023-11-19T01:29:35,650 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollRegionServerProcedureManager(126): Attempting to run a roll log procedure for backup. 2023-11-19T01:29:35,650 INFO [zk-event-processor-pool-0 {}] regionserver.LogRollBackupSubprocedure(56): Constructing a LogRollBackupSubprocedure. 2023-11-19T01:29:35,650 WARN [zk-event-processor-pool-0 {}] procedure.ProcedureMember(133): A completed old subproc rolllog is still present, removing 2023-11-19T01:29:35,650 DEBUG [zk-event-processor-pool-0 {}] procedure.ProcedureMember(140): Submitting new Subprocedure:rolllog 2023-11-19T01:29:35,650 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(151): Starting subprocedure 'rolllog' with timeout 60000ms 2023-11-19T01:29:35,650 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(107): Scheduling process timer to run in: 60000 ms 2023-11-19T01:29:35,651 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(159): Subprocedure 'rolllog' starting 'acquire' stage 2023-11-19T01:29:35,651 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(161): Subprocedure 'rolllog' locally acquired 2023-11-19T01:29:35,651 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(242): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure (rolllog) in zk 2023-11-19T01:29:35,654 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(250): Watch for global barrier reached:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:35,654 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/acquired/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:35,654 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:29:35,654 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:29:35,654 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(166): Subprocedure 'rolllog' coordinator notified of 'acquire', waiting on 'reached' or 'abort' from coordinator 2023-11-19T01:29:35,654 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:29:35,655 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:29:35,655 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:35,655 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:35,656 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:29:35,656 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(291): member: 'jenkins-hbase6.apache.org,44997,1700357176784' joining acquired barrier for procedure 'rolllog' on coordinator 2023-11-19T01:29:35,656 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(300): Waiting on: java.util.concurrent.CountDownLatch@240d2566[Count = 0] remaining members to acquire global barrier 2023-11-19T01:29:35,656 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(207): Procedure 'rolllog' starting 'in-barrier' execution. 2023-11-19T01:29:35,656 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(116): Creating reached barrier zk node:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:35,657 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:35,657 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(128): Received reached global barrier:/1/rolllog-proc/reached/rolllog 2023-11-19T01:29:35,657 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(180): Subprocedure 'rolllog' received 'reached' from coordinator. 2023-11-19T01:29:35,657 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] regionserver.LogRollBackupSubprocedurePool(84): Waiting for backup procedure to finish. 2023-11-19T01:29:35,658 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(211): Waiting for all members to 'release' 2023-11-19T01:29:35,658 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(75): DRPC started: jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:35,658 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(92): Trying to roll log in backup subprocedure, current log number: 1700357361878 highest: 1700357361878 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:35,658 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784:(num 1700357361878) roll requested 2023-11-19T01:29:35,669 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357375658, exclude list is [], retry=0 2023-11-19T01:29:35,673 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:29:35,674 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357361878 with entries=13, filesize=3.13 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357375658 2023-11-19T01:29:35,675 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:29:35,675 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357361878 is not closed yet, will try archiving it next time 2023-11-19T01:29:35,675 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(809): Archiving hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357352327 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/oldWALs/jenkins-hbase6.apache.org%2C44997%2C1700357176784.1700357352327 2023-11-19T01:29:35,675 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(197): WAL AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta:.meta(num 1700357361894) roll requested 2023-11-19T01:29:35,719 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(532): When create output stream for /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357375675.meta, exclude list is [], retry=0 2023-11-19T01:29:35,723 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(810): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK] 2023-11-19T01:29:35,724 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(828): Rolled WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357361894.meta with entries=12, filesize=2.74 KB; new WAL /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357375675.meta 2023-11-19T01:29:35,724 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(916): Create new AsyncFSWAL writer with pipeline: [DatanodeInfoWithStorage[127.0.0.1:43079,DS-0e894dbf-f050-45c4-aafb-6175da034d96,DISK]] 2023-11-19T01:29:35,724 DEBUG [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractFSWAL(742): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/WALs/jenkins-hbase6.apache.org,44997,1700357176784/jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta.1700357361894.meta is not closed yet, will try archiving it next time 2023-11-19T01:29:35,727 DEBUG [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(99): log roll took 69 2023-11-19T01:29:35,727 INFO [rs(jenkins-hbase6.apache.org,44997,1700357176784)-backup-pool-0 {}] regionserver.LogRollBackupSubprocedure$RSRollLogTask(100): After roll log in backup subprocedure, current log number: 1700357375658 on jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:35,734 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(182): Subprocedure 'rolllog' locally completed 2023-11-19T01:29:35,734 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.ZKProcedureMemberRpcs(267): Marking procedure 'rolllog' completed for member 'jenkins-hbase6.apache.org,44997,1700357176784' in zk 2023-11-19T01:29:35,737 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(187): Subprocedure 'rolllog' has notified controller of completion 2023-11-19T01:29:35,737 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:29:35,737 DEBUG [member: 'jenkins-hbase6.apache.org,44997,1700357176784' subprocedure-pool-0 {}] procedure.Subprocedure(212): Subprocedure 'rolllog' completed. 2023-11-19T01:29:35,738 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/reached/rolllog/jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:35,738 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:29:35,738 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:29:35,738 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:29:35,739 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:29:35,739 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:35,739 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:35,740 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:29:35,740 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:35,740 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:35,741 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(218): Finished data from procedure 'rolllog' member 'jenkins-hbase6.apache.org,44997,1700357176784': 2023-11-19T01:29:35,741 DEBUG [zk-event-processor-pool-0 {}] procedure.Procedure(321): Member: 'jenkins-hbase6.apache.org,44997,1700357176784' released barrier for procedure'rolllog', counting down latch. Waiting for 0 more 2023-11-19T01:29:35,741 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(216): Procedure 'rolllog' execution completed 2023-11-19T01:29:35,741 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(225): Running finish phase. 2023-11-19T01:29:35,741 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.Procedure(275): Finished coordinator procedure - removing self from list of running procedures 2023-11-19T01:29:35,741 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureCoordinator(162): Attempting to clean out zk node for op:rolllog 2023-11-19T01:29:35,741 INFO [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] procedure.ZKProcedureUtil(265): Clearing all znodes for procedure rolllogincluding nodes /1/rolllog-proc/acquired /1/rolllog-proc/reached /1/rolllog-proc/abort 2023-11-19T01:29:35,743 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureCoordinator$1(194): Node created: /1/rolllog-proc/abort/rolllog 2023-11-19T01:29:35,743 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(77): Received created event:/1/rolllog-proc/abort/rolllog 2023-11-19T01:29:35,743 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(225): Current zk system: 2023-11-19T01:29:35,743 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(227): |-/1/rolllog-proc 2023-11-19T01:29:35,743 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:29:35,743 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-abort 2023-11-19T01:29:35,743 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:29:35,743 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:29:35,744 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:35,744 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(316): Aborting procedure member for znode /1/rolllog-proc/abort/rolllog 2023-11-19T01:29:35,744 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-acquired 2023-11-19T01:29:35,744 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:35,745 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:35,745 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-reached 2023-11-19T01:29:35,745 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |----rolllog 2023-11-19T01:29:35,746 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureUtil(244): |-------jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:35,751 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(104): Received procedure start children changed event: /1/rolllog-proc/acquired 2023-11-19T01:29:35,751 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/1/rolllog-proc/acquired' 2023-11-19T01:29:35,751 DEBUG [(jenkins-hbase6.apache.org,44889,1700357173572)-proc-coordinator-pool-0 {}] errorhandling.TimeoutExceptionInjector(87): Marking timer as complete - no error notifications will be received for this timer. 2023-11-19T01:29:35,751 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(143): Done waiting - exec procedure for rolllog 2023-11-19T01:29:35,751 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.LogRollMasterProcedureManager(144): Distributed roll log procedure is successful! 2023-11-19T01:29:35,751 INFO [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs$1(107): Received procedure abort children changed event: /1/rolllog-proc/abort 2023-11-19T01:29:35,751 DEBUG [zk-event-processor-pool-0 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/1/rolllog-proc/abort' 2023-11-19T01:29:35,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1263): Checking to see if procedure from request:rolllog-proc is done 2023-11-19T01:29:35,855 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357375854_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } 2023-11-19T01:29:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357375855 (current time:1700357375855). 2023-11-19T01:29:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:29:35,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357375854_ns2_test-17003571846791 VERSION not specified, setting to 2 2023-11-19T01:29:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@261bf3cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:35,865 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:40120, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:35,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:29:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=170, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1700357375854_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } 2023-11-19T01:29:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357375854_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 }, snapshot procedure id = 170 2023-11-19T01:29:35,869 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1700357375854_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:29:35,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=170 2023-11-19T01:29:35,869 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1700357375854_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:29:35,872 WARN [PEWorker-2 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357375854_ns2_test-17003571846791 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:29:35,874 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1700357375854_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:29:35,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=170 2023-11-19T01:29:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=170 2023-11-19T01:29:36,280 INFO [PEWorker-2 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1700357375854_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:29:36,280 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE; SnapshotRegionProcedure a169e45a4f1d98e65b6a1dada440fc40}] 2023-11-19T01:29:36,282 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=171, ppid=170, state=RUNNABLE; SnapshotRegionProcedure a169e45a4f1d98e65b6a1dada440fc40 2023-11-19T01:29:36,433 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:36,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=171 2023-11-19T01:29:36,433 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40. 2023-11-19T01:29:36,434 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.HRegion(2520): Flush status journal for a169e45a4f1d98e65b6a1dada440fc40: 2023-11-19T01:29:36,434 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40. for snapshot_1700357375854_ns2_test-17003571846791 completed. 2023-11-19T01:29:36,434 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(240): Storing 'ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40.' region-info for snapshot=snapshot_1700357375854_ns2_test-17003571846791 2023-11-19T01:29:36,434 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:29:36,434 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(255): Adding snapshot references for [hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40/f/9301f8758b1e47ab85817026d7156af0_SeqId_4_] hfiles 2023-11-19T01:29:36,434 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] snapshot.SnapshotManifest(264): Adding reference for file (1/1): hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40/f/9301f8758b1e47ab85817026d7156af0_SeqId_4_ for snapshot=snapshot_1700357375854_ns2_test-17003571846791 2023-11-19T01:29:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=170 2023-11-19T01:29:36,839 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40. 2023-11-19T01:29:36,839 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-0 {event_type=RS_SNAPSHOT_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=171 2023-11-19T01:29:36,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=171 2023-11-19T01:29:36,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357375854_ns2_test-17003571846791 on region a169e45a4f1d98e65b6a1dada440fc40 2023-11-19T01:29:36,841 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=171, ppid=170, state=RUNNABLE; SnapshotRegionProcedure a169e45a4f1d98e65b6a1dada440fc40 2023-11-19T01:29:36,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=171, resume processing ppid=170 2023-11-19T01:29:36,843 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1700357375854_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:29:36,843 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=171, ppid=170, state=SUCCESS; SnapshotRegionProcedure a169e45a4f1d98e65b6a1dada440fc40 in 561 msec 2023-11-19T01:29:36,843 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1700357375854_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:29:36,843 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1700357375854_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:29:36,844 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357375854_ns2_test-17003571846791 2023-11-19T01:29:36,844 DEBUG [PEWorker-5 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357375854_ns2_test-17003571846791 2023-11-19T01:29:36,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=170 2023-11-19T01:29:37,252 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1700357375854_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:29:37,257 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1700357375854_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:29:37,257 DEBUG [PEWorker-5 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357375854_ns2_test-17003571846791 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357375854_ns2_test-17003571846791 2023-11-19T01:29:37,258 INFO [PEWorker-5 {}] procedure.SnapshotProcedure(134): pid=170, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1700357375854_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:29:37,258 DEBUG [PEWorker-5 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357375854_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 }, snapshot procedure id = 170 2023-11-19T01:29:37,259 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1413): Finished pid=170, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=170, snapshot={ ss=snapshot_1700357375854_ns2_test-17003571846791 table=ns2:test-17003571846791 type=FLUSH ttl=0 } in 1.3920 sec 2023-11-19T01:29:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=170 2023-11-19T01:29:37,974 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1701): Client=jenkins//172.31.2.20 snapshot request for:{ ss=snapshot_1700357377974_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } 2023-11-19T01:29:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(331): Creation time not specified, setting to:1700357377974 (current time:1700357377974). 2023-11-19T01:29:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(345): Snapshot current TTL value: 0 resetting it to default value: 0 2023-11-19T01:29:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotDescriptionUtils(354): Snapshot snapshot_1700357377974_ns3_test-17003571846792 VERSION not specified, setting to 2 2023-11-19T01:29:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71b96b48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=null 2023-11-19T01:29:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.RpcConnection(127): Using SIMPLE authentication for service=ClientService, sasl=false 2023-11-19T01:29:37,983 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(478): Connection from 172.31.2.20:40136, version=2.6.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2023-11-19T01:29:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(806): No existing snapshot, attempting snapshot... 2023-11-19T01:29:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure2.ProcedureExecutor(1032): Stored pid=172, state=RUNNABLE:SNAPSHOT_PREPARE; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1700357377974_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } 2023-11-19T01:29:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(1440): register snapshot={ ss=snapshot_1700357377974_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 }, snapshot procedure id = 172 2023-11-19T01:29:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=172 2023-11-19T01:29:37,987 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1700357377974_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_PREPARE 2023-11-19T01:29:37,988 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_PRE_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1700357377974_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_PRE_OPERATION 2023-11-19T01:29:37,990 WARN [PEWorker-4 {}] snapshot.SnapshotManager(598): Unable to retrieve ACL status for path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot, current working dir path: hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357377974_ns3_test-17003571846792 org.apache.hadoop.hdfs.protocol.AclException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at sun.reflect.GeneratedConstructorAccessor130.newInstance(Unknown Source) ~[?:?] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_362] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_362] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2763) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2306) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$51.doCall(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.getAclStatus(DistributedFileSystem.java:2303) ~[hadoop-hdfs-client-2.10.2.jar:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.updateWorkingDirAclsIfRequired(SnapshotManager.java:596) ~[classes/:?] at org.apache.hadoop.hbase.master.snapshot.SnapshotManager.prepareWorkingDirectory(SnapshotManager.java:569) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.preSnapshot(SnapshotProcedure.java:302) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:143) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.SnapshotProcedure.executeFromState(SnapshotProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:922) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1649) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1395) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:75) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:1961) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:1988) ~[hbase-procedure-2.6.0-SNAPSHOT.jar:2.6.0-SNAPSHOT] Caused by: org.apache.hadoop.ipc.RemoteException: The ACL operation has been rejected. Support for ACLs has been disabled by setting dfs.namenode.acls.enabled to false. at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.checkAclsConfigFlag(FSDirAclOp.java:197) at org.apache.hadoop.hdfs.server.namenode.FSDirAclOp.getAclStatus(FSDirAclOp.java:150) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getAclStatus(FSNamesystem.java:6799) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getAclStatus(NameNodeRpcServer.java:2085) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getAclStatus(ClientNamenodeProtocolServerSideTranslatorPB.java:1378) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:528) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:498) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1038) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1003) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:931) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1938) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2855) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1560) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1506) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1403) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:230) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy40.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getAclStatus(ClientNamenodeProtocolTranslatorPB.java:1385) ~[hadoop-hdfs-client-2.10.2.jar:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:433) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:166) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:158) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:96) ~[hadoop-common-2.10.2.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:362) ~[hadoop-common-2.10.2.jar:?] at com.sun.proxy.$Proxy41.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at sun.reflect.GeneratedMethodAccessor56.invoke(Unknown Source) ~[?:?] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at com.sun.proxy.$Proxy44.getAclStatus(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.getAclStatus(DFSClient.java:2761) ~[hadoop-hdfs-client-2.10.2.jar:?] ... 17 more 2023-11-19T01:29:37,993 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_WRITE_SNAPSHOT_INFO, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1700357377974_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_WRITE_SNAPSHOT_INFO 2023-11-19T01:29:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=172 2023-11-19T01:29:38,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=172 2023-11-19T01:29:38,398 INFO [PEWorker-4 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_SNAPSHOT_ONLINE_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1700357377974_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_ONLINE_REGIONS 2023-11-19T01:29:38,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1684): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE; SnapshotRegionProcedure 09936870cc2abdecec74e26057d016f7}] 2023-11-19T01:29:38,400 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=173, ppid=172, state=RUNNABLE; SnapshotRegionProcedure 09936870cc2abdecec74e26057d016f7 2023-11-19T01:29:38,551 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(710): New admin connection to jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:38,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44997 {}] regionserver.RSRpcServices(3926): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SnapshotRegionCallable, pid=173 2023-11-19T01:29:38,551 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.SnapshotRegionCallable(50): Starting snapshot operation on ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7. 2023-11-19T01:29:38,552 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.HRegion(2520): Flush status journal for 09936870cc2abdecec74e26057d016f7: 2023-11-19T01:29:38,552 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.SnapshotRegionCallable(75): Snapshotting region ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7. for snapshot_1700357377974_ns3_test-17003571846792 completed. 2023-11-19T01:29:38,552 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] snapshot.SnapshotManifest(240): Storing 'ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7.' region-info for snapshot=snapshot_1700357377974_ns3_test-17003571846792 2023-11-19T01:29:38,552 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] snapshot.SnapshotManifest(245): Creating references for hfiles 2023-11-19T01:29:38,552 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] snapshot.SnapshotManifest(255): Adding snapshot references for [] hfiles 2023-11-19T01:29:38,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=172 2023-11-19T01:29:38,957 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.SnapshotRegionCallable(78): Closing snapshot operation on ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7. 2023-11-19T01:29:38,957 DEBUG [RS_SNAPSHOT_OPERATIONS-regionserver/jenkins-hbase6:0-1 {event_type=RS_SNAPSHOT_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(61): Successfully complete execution of pid=173 2023-11-19T01:29:38,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.HMaster(4026): Remote procedure done, pid=173 2023-11-19T01:29:38,958 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] procedure.SnapshotRegionProcedure(130): finish snapshot snapshot_1700357377974_ns3_test-17003571846792 on region 09936870cc2abdecec74e26057d016f7 2023-11-19T01:29:38,959 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(727): Took xlock for pid=173, ppid=172, state=RUNNABLE; SnapshotRegionProcedure 09936870cc2abdecec74e26057d016f7 2023-11-19T01:29:38,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1827): Finished subprocedure pid=173, resume processing ppid=172 2023-11-19T01:29:38,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1413): Finished pid=173, ppid=172, state=SUCCESS; SnapshotRegionProcedure 09936870cc2abdecec74e26057d016f7 in 562 msec 2023-11-19T01:29:38,961 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_SNAPSHOT_SPLIT_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1700357377974_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_SPLIT_REGIONS 2023-11-19T01:29:38,962 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_SNAPSHOT_MOB_REGION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1700357377974_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_SNAPSHOT_MOB_REGION 2023-11-19T01:29:38,962 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_CONSOLIDATE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1700357377974_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_CONSOLIDATE_SNAPSHOT 2023-11-19T01:29:38,962 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifest(489): Convert to Single Snapshot Manifest for snapshot_1700357377974_ns3_test-17003571846792 2023-11-19T01:29:38,963 DEBUG [PEWorker-3 {}] snapshot.SnapshotManifestV1(126): No regions under directory:hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357377974_ns3_test-17003571846792 2023-11-19T01:29:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=172 2023-11-19T01:29:39,371 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_VERIFIER_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1700357377974_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_VERIFIER_SNAPSHOT 2023-11-19T01:29:39,375 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_COMPLETE_SNAPSHOT, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1700357377974_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_COMPLETE_SNAPSHOT 2023-11-19T01:29:39,375 DEBUG [PEWorker-3 {}] snapshot.SnapshotDescriptionUtils(430): Sentinel is done, just moving the snapshot from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/.tmp/snapshot_1700357377974_ns3_test-17003571846792 to hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357377974_ns3_test-17003571846792 2023-11-19T01:29:39,376 INFO [PEWorker-3 {}] procedure.SnapshotProcedure(134): pid=172, state=RUNNABLE:SNAPSHOT_POST_OPERATION, locked=true; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1700357377974_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } execute state=SNAPSHOT_POST_OPERATION 2023-11-19T01:29:39,376 DEBUG [PEWorker-3 {}] snapshot.SnapshotManager(1446): unregister snapshot={ ss=snapshot_1700357377974_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 }, snapshot procedure id = 172 2023-11-19T01:29:39,377 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1413): Finished pid=172, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.SnapshotProcedure, id=172, snapshot={ ss=snapshot_1700357377974_ns3_test-17003571846792 table=ns3:test-17003571846792 type=FLUSH ttl=0 } in 1.3920 sec 2023-11-19T01:29:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(1304): Checking to see if procedure is done pid=172 2023-11-19T01:29:40,092 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(171): snapshot copy for backup_1700357371502 2023-11-19T01:29:40,092 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(68): Snapshot copy is starting. 2023-11-19T01:29:40,092 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(80): There are 2 snapshots to be copied. 2023-11-19T01:29:40,092 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357371502_ns2:test-17003571846791 2023-11-19T01:29:40,092 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357375854_ns2_test-17003571846791 to hdfs://localhost:42739/backupUT/backup_1700357371502/ns2/test-17003571846791/ 2023-11-19T01:29:40,092 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:29:40,115 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:29:40,115 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,44889,1700357173572_-236689627_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42739/backupUT/backup_1700357371502/ns2/test-17003571846791, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42739/backupUT/backup_1700357371502/ns2/test-17003571846791/.hbase-snapshot/.tmp/snapshot_1700357375854_ns2_test-17003571846791 2023-11-19T01:29:40,115 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af, snapshotDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357375854_ns2_test-17003571846791. 2023-11-19T01:29:40,122 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357375854_ns2_test-17003571846791 to hdfs://localhost:42739/backupUT/backup_1700357371502/ns2/test-17003571846791/.hbase-snapshot/.tmp/snapshot_1700357375854_ns2_test-17003571846791 2023-11-19T01:29:40,544 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:40,545 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:40,545 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:40,545 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:41,565 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-6767851071819984705.jar 2023-11-19T01:29:41,565 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:41,566 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:41,566 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:41,566 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:41,567 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:41,567 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:41,567 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:41,568 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:41,568 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:41,569 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:29:41,569 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:29:41,570 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:29:41,570 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:29:41,571 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:29:41,571 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:29:41,571 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:29:41,572 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:29:41,572 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:29:41,572 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:29:41,573 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:29:41,573 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:29:41,574 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:41,574 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:41,575 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:41,575 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:41,575 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:41,576 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:41,576 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:41,619 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357375854_ns2_test-17003571846791' hfile list 2023-11-19T01:29:41,620 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(674): export split=0 size=8.5 K 2023-11-19T01:29:41,692 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357381655/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:29:41,692 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357381655/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:29:41,743 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(228): Using bufferSize=128 M 2023-11-19T01:29:41,767 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(454): copy completed for input=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40/f/9301f8758b1e47ab85817026d7156af0_SeqId_4_ output=hdfs://localhost:42739/backupUT/backup_1700357371502/ns2/test-17003571846791/archive/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40/f/9301f8758b1e47ab85817026d7156af0_SeqId_4_ 2023-11-19T01:29:41,767 INFO [LocalJobRunner Map Task Executor #0 {}] snapshot.ExportSnapshot$ExportMapper(456): size=8724 (8.5 K) time=0sec 8.320M/sec 2023-11-19T01:29:42,722 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:29:42,723 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:29:42,728 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357375854_ns2_test-17003571846791 2023-11-19T01:29:42,728 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357375854_ns2_test-17003571846791 finished. 2023-11-19T01:29:42,728 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(105): Setting snapshot copy job name to : Full-Backup_backup_1700357371502_ns3:test-17003571846792 2023-11-19T01:29:42,729 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(109): Copy snapshot snapshot_1700357377974_ns3_test-17003571846792 to hdfs://localhost:42739/backupUT/backup_1700357371502/ns3/test-17003571846792/ 2023-11-19T01:29:42,729 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.MapReduceBackupCopyJob(373): Doing SNAPSHOT_COPY 2023-11-19T01:29:42,754 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1003): inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af 2023-11-19T01:29:42,754 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1004): outputFs=DFS[DFSClient[clientName=DFSClient_hb_m_jenkins-hbase6.apache.org,44889,1700357173572_2080279249_16, ugi=jenkins (auth:SIMPLE)]], outputRoot=hdfs://localhost:42739/backupUT/backup_1700357371502/ns3/test-17003571846792, skipTmp=false, initialOutputSnapshotDir=hdfs://localhost:42739/backupUT/backup_1700357371502/ns3/test-17003571846792/.hbase-snapshot/.tmp/snapshot_1700357377974_ns3_test-17003571846792 2023-11-19T01:29:42,754 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1009): Verify snapshot source, inputFs=hdfs://localhost:42739, inputRoot=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af, snapshotDir=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357377974_ns3_test-17003571846792. 2023-11-19T01:29:42,759 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1068): Copy Snapshot Manifest from hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/.hbase-snapshot/snapshot_1700357377974_ns3_test-17003571846792 to hdfs://localhost:42739/backupUT/backup_1700357371502/ns3/test-17003571846792/.hbase-snapshot/.tmp/snapshot_1700357377974_ns3_test-17003571846792 2023-11-19T01:29:43,180 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.HConstants, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-common/target/hbase-common-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:43,180 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol/target/hbase-protocol-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:43,181 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-protocol-shaded/target/hbase-protocol-shaded-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:43,181 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.client.Put, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-client/target/hbase-client-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:44,178 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.ipc.RpcServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop-4512630208000715299.jar 2023-11-19T01:29:44,179 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.CompatibilityFactory, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop-compat/target/hbase-hadoop-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:44,179 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.JobUtil, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-hadoop2-compat/target/hbase-hadoop2-compat-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:44,179 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.mapreduce.TableMapper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-mapreduce/target/hbase-mapreduce-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:44,180 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.impl.FastLongHistogram, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics/target/hbase-metrics-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:44,180 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.metrics.Snapshot, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-metrics-api/target/hbase-metrics-api-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:44,180 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.replication.ReplicationUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-replication/target/hbase-replication-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:44,181 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.http.HttpServer, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-http/target/hbase-http-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:44,181 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.procedure2.Procedure, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-procedure/target/hbase-procedure-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:44,181 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.zookeeper.ZKWatcher, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-zookeeper/target/hbase-zookeeper-2.6.0-SNAPSHOT.jar 2023-11-19T01:29:44,182 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.common.collect.Lists, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-miscellaneous/4.1.5/hbase-shaded-miscellaneous-4.1.5.jar 2023-11-19T01:29:44,183 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.gson.GsonBuilder, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-gson/4.1.5/hbase-shaded-gson-4.1.5.jar 2023-11-19T01:29:44,183 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-protobuf/4.1.5/hbase-shaded-protobuf-4.1.5.jar 2023-11-19T01:29:44,183 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hbase.thirdparty.io.netty.channel.Channel, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-shaded-netty/4.1.5/hbase-shaded-netty-4.1.5.jar 2023-11-19T01:29:44,184 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.hbase.unsafe.HBasePlatformDependent, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hbase/thirdparty/hbase-unsafe/4.1.5/hbase-unsafe-4.1.5.jar 2023-11-19T01:29:44,184 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.zookeeper.ZooKeeper, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/zookeeper/zookeeper/3.8.3/zookeeper-3.8.3.jar 2023-11-19T01:29:44,184 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.google.protobuf.Message, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar 2023-11-19T01:29:44,185 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class com.codahale.metrics.MetricRegistry, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/dropwizard/metrics/metrics-core/3.2.6/metrics-core-3.2.6.jar 2023-11-19T01:29:44,185 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.commons.lang3.ArrayUtils, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/commons/commons-lang3/3.9/commons-lang3-3.9.jar 2023-11-19T01:29:44,185 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.api.trace.Span, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-api/1.15.0/opentelemetry-api-1.15.0.jar 2023-11-19T01:29:44,186 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.semconv.trace.attributes.SemanticAttributes, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-semconv/1.15.0-alpha/opentelemetry-semconv-1.15.0-alpha.jar 2023-11-19T01:29:44,186 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class io.opentelemetry.context.Context, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/io/opentelemetry/opentelemetry-context/1.15.0/opentelemetry-context-1.15.0.jar 2023-11-19T01:29:44,187 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:44,187 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:44,188 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.input.TextInputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:44,188 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.LongWritable, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:44,188 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.io.Text, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-common/2.10.2/hadoop-common-2.10.2.jar 2023-11-19T01:29:44,189 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.output.TextOutputFormat, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:44,190 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] mapreduce.TableMapReduceUtil(868): For class org.apache.hadoop.mapreduce.lib.partition.HashPartitioner, using jar /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-mapreduce-client-core/2.10.2/hadoop-mapreduce-client-core-2.10.2.jar 2023-11-19T01:29:44,232 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(581): Loading Snapshot 'snapshot_1700357377974_ns3_test-17003571846792' hfile list 2023-11-19T01:29:44,300 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] fs.FileUtil(888): Command 'ln -s /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357384267/libjars /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*' failed 1 with: ln: failed to create symbolic link '/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/*': No such file or directory 2023-11-19T01:29:44,300 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] mapred.LocalDistributedCacheManager(202): Failed to create symlink: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/mapreduce.cluster.local.dir/1700357384267/libjars <- /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/libjars/* 2023-11-19T01:29:45,327 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1125): Finalize the Snapshot Export 2023-11-19T01:29:45,328 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1136): Verify snapshot integrity 2023-11-19T01:29:45,332 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] snapshot.ExportSnapshot(1140): Export Completed: snapshot_1700357377974_ns3_test-17003571846792 2023-11-19T01:29:45,332 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.FullTableBackupClient(121): Snapshot copy snapshot_1700357377974_ns3_test-17003571846792 finished. 2023-11-19T01:29:45,332 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1034): ns2:test-17003571846791 2023-11-19T01:29:45,332 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1034): ns3:test-17003571846792 2023-11-19T01:29:45,338 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357371502 2023-11-19T01:29:45,338 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:29:45,743 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42739/backupUT/backup_1700357371502/.backup.manifest 2023-11-19T01:29:45,743 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(277): Getting the direct ancestors of the current backup backup_1700357371502 2023-11-19T01:29:45,743 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManager(283): Current backup is a full backup, no direct ancestor for it. 2023-11-19T01:29:46,148 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(484): Manifest file stored to hdfs://localhost:42739/backupUT/backup_1700357371502/.backup.manifest 2023-11-19T01:29:46,148 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(358): Backup backup_1700357371502 finished: type=FULL,tablelist=ns2:test-17003571846791;ns3:test-17003571846792,targetRootDir=hdfs://localhost:42739/backupUT,startts=1700357375640,completets=1700357385338,bytescopied=0 2023-11-19T01:29:46,148 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(138): Trying to delete snapshot for full backup. 2023-11-19T01:29:46,148 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357375854_ns2_test-17003571846791 2023-11-19T01:29:46,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357375854_ns2_test-17003571846791" 2023-11-19T01:29:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357375854_ns2_test-17003571846791 2023-11-19T01:29:46,152 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357375854_ns2_test-17003571846791 for backup backup_1700357371502 succeeded. 2023-11-19T01:29:46,152 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(143): Trying to delete snapshot: snapshot_1700357377974_ns3_test-17003571846792 2023-11-19T01:29:46,153 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_1700357377974_ns3_test-17003571846792" 2023-11-19T01:29:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_1700357377974_ns3_test-17003571846792 2023-11-19T01:29:46,155 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(148): Deleting the snapshot snapshot_1700357377974_ns3_test-17003571846792 for backup backup_1700357371502 succeeded. 2023-11-19T01:29:46,156 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1572): Deleting snapshot_backup_system from the system 2023-11-19T01:29:46,160 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] master.MasterRpcServices(763): Client=jenkins//172.31.2.20 delete name: "snapshot_backup_system" 2023-11-19T01:29:46,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44889 {}] snapshot.SnapshotManager(380): Deleting snapshot: snapshot_backup_system 2023-11-19T01:29:46,163 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(1577): Done deleting backup system table snapshot 2023-11-19T01:29:46,164 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupSystemTable(657): Finish backup exclusive operation 2023-11-19T01:29:46,170 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.TableBackupClient(376): Backup backup_1700357371502 completed. 2023-11-19T01:29:46,170 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:46,179 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(101): HBase table ns2:table2 does not exist. It will be created during restore process 2023-11-19T01:29:46,180 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(101): HBase table ns3:table3_restore does not exist. It will be created during restore process 2023-11-19T01:29:46,180 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357371502/.backup.manifest 2023-11-19T01:29:46,182 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357371502/.backup.manifest 2023-11-19T01:29:46,182 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357371502/.backup.manifest 2023-11-19T01:29:46,184 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357371502/.backup.manifest 2023-11-19T01:29:46,184 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(413): Loading manifest from: hdfs://localhost:42739/backupUT/backup_1700357371502/.backup.manifest 2023-11-19T01:29:46,186 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] impl.BackupManifest(443): Loaded manifest instance from manifest file: /backupUT/backup_1700357371502/.backup.manifest 2023-11-19T01:29:46,186 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] impl.RestoreTablesClient(148): Restoring 'faketable1' to 'ns2:table2' from full backup image hdfs://localhost:42739/backupUT/backup_1700357371502/default/faketable1 2023-11-19T01:29:46,203 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(175): after: backup.TestFullRestore#testFullRestoreMultipleDNE Thread=604 (was 605), OpenFileDescriptor=723 (was 713) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=60000 (was 60000), SystemLoadAverage=128 (was 128), ProcessCount=166 (was 166), AvailableMemoryMB=3380 (was 3436) 2023-11-19T01:29:46,203 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.ResourceChecker(130): Thread=604 is superior to 500 2023-11-19T01:29:46,209 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2023-11-19T01:29:46,209 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:46,209 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2023-11-19T01:29:46,209 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] util.JVMClusterUtil(257): Found active master hash=719250638, stopped=false 2023-11-19T01:29:46,210 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] master.ServerManager(896): Cluster shutdown requested of master=jenkins-hbase6.apache.org,44889,1700357173572 2023-11-19T01:29:46,216 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] procedure2.ProcedureExecutor(630): Stopping 2023-11-19T01:29:46,217 DEBUG [Listener at jenkins-hbase6.apache.org/35909 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:46,217 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase6.apache.org,44997,1700357176784' ***** 2023-11-19T01:29:46,217 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] regionserver.HRegionServer(2531): STOPPED: Shutdown requested 2023-11-19T01:29:46,217 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-19T01:29:46,226 INFO [regionserver/jenkins-hbase6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2023-11-19T01:29:46,230 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2023-11-19T01:29:46,231 INFO [RS:0;jenkins-hbase6:44997 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@52704e90{regionserver,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/regionserver} 2023-11-19T01:29:46,235 INFO [RS:0;jenkins-hbase6:44997 {}] server.AbstractConnector(383): Stopped ServerConnector@65b85596{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-19T01:29:46,235 INFO [RS:0;jenkins-hbase6:44997 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-19T01:29:46,236 INFO [RS:0;jenkins-hbase6:44997 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@5ab6f613{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-19T01:29:46,236 INFO [RS:0;jenkins-hbase6:44997 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@50332b50{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop.log.dir/,STOPPED} 2023-11-19T01:29:46,239 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HeapMemoryManager(220): Stopping 2023-11-19T01:29:46,239 INFO [RS:0;jenkins-hbase6:44997 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2023-11-19T01:29:46,239 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.LogRollRegionServerProcedureManager(106): Stopping RegionServerBackupManager gracefully. 2023-11-19T01:29:46,239 INFO [RS:0;jenkins-hbase6:44997 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2023-11-19T01:29:46,239 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(3528): Received CLOSE for fddaaf63127dc05fabd03e1195f3962c 2023-11-19T01:29:46,239 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(3528): Received CLOSE for 09936870cc2abdecec74e26057d016f7 2023-11-19T01:29:46,239 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(3528): Received CLOSE for f8f5111e949766749b5234926ea710eb 2023-11-19T01:29:46,239 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(3528): Received CLOSE for 322466352cac3ade468ab2ce8e706acc 2023-11-19T01:29:46,239 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(3528): Received CLOSE for b696d85cf645e95a934c0b80f486a6f6 2023-11-19T01:29:46,239 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(3528): Received CLOSE for e7af7e5272e1026697882da645f6c4df 2023-11-19T01:29:46,239 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(3528): Received CLOSE for a169e45a4f1d98e65b6a1dada440fc40 2023-11-19T01:29:46,239 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase6.apache.org,44997,1700357176784 2023-11-19T01:29:46,240 DEBUG [RS:0;jenkins-hbase6:44997 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:46,240 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2023-11-19T01:29:46,240 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2023-11-19T01:29:46,240 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing fddaaf63127dc05fabd03e1195f3962c, disabling compactions & flushes 2023-11-19T01:29:46,240 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2023-11-19T01:29:46,240 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c. 2023-11-19T01:29:46,240 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(3528): Received CLOSE for 1588230740 2023-11-19T01:29:46,240 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c. 2023-11-19T01:29:46,240 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c. after waiting 0 ms 2023-11-19T01:29:46,240 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c. 2023-11-19T01:29:46,241 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1579): Waiting on 8 regions to close 2023-11-19T01:29:46,241 DEBUG [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1583): Online Regions={fddaaf63127dc05fabd03e1195f3962c=ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c., 09936870cc2abdecec74e26057d016f7=ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7., f8f5111e949766749b5234926ea710eb=hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb., 322466352cac3ade468ab2ce8e706acc=backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc., b696d85cf645e95a934c0b80f486a6f6=test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6., 1588230740=hbase:meta,,1.1588230740, e7af7e5272e1026697882da645f6c4df=backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df., a169e45a4f1d98e65b6a1dada440fc40=ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40.} 2023-11-19T01:29:46,244 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1668): Closing 1588230740, disabling compactions & flushes 2023-11-19T01:29:46,244 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1690): Closing region hbase:meta,,1.1588230740 2023-11-19T01:29:46,244 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2023-11-19T01:29:46,244 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1778): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2023-11-19T01:29:46,244 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1788): Updates disabled for region hbase:meta,,1.1588230740 2023-11-19T01:29:46,244 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2819): Flushing 1588230740 3/3 column families, dataSize=38.00 KB heapSize=66.30 KB 2023-11-19T01:29:46,244 DEBUG [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1609): Waiting on 09936870cc2abdecec74e26057d016f7, 1588230740, 322466352cac3ade468ab2ce8e706acc, a169e45a4f1d98e65b6a1dada440fc40, b696d85cf645e95a934c0b80f486a6f6, e7af7e5272e1026697882da645f6c4df, f8f5111e949766749b5234926ea710eb, fddaaf63127dc05fabd03e1195f3962c 2023-11-19T01:29:46,247 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns4/test-17003571846793/fddaaf63127dc05fabd03e1195f3962c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2023-11-19T01:29:46,247 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:29:46,248 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c. 2023-11-19T01:29:46,248 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for fddaaf63127dc05fabd03e1195f3962c: 2023-11-19T01:29:46,248 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns4:test-17003571846793,,1700357190750.fddaaf63127dc05fabd03e1195f3962c. 2023-11-19T01:29:46,248 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 09936870cc2abdecec74e26057d016f7, disabling compactions & flushes 2023-11-19T01:29:46,248 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7. 2023-11-19T01:29:46,248 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7. 2023-11-19T01:29:46,248 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7. after waiting 0 ms 2023-11-19T01:29:46,249 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7. 2023-11-19T01:29:46,251 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns3/test-17003571846792/09936870cc2abdecec74e26057d016f7/recovered.edits/6.seqid, newMaxSeqId=6, maxSeqId=1 2023-11-19T01:29:46,252 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:29:46,252 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7. 2023-11-19T01:29:46,252 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 09936870cc2abdecec74e26057d016f7: 2023-11-19T01:29:46,253 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns3:test-17003571846792,,1700357326728.09936870cc2abdecec74e26057d016f7. 2023-11-19T01:29:46,253 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing f8f5111e949766749b5234926ea710eb, disabling compactions & flushes 2023-11-19T01:29:46,253 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb. 2023-11-19T01:29:46,253 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb. 2023-11-19T01:29:46,253 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb. after waiting 0 ms 2023-11-19T01:29:46,253 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb. 2023-11-19T01:29:46,253 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing f8f5111e949766749b5234926ea710eb 1/1 column families, dataSize=249 B heapSize=1.02 KB 2023-11-19T01:29:46,266 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/namespace/f8f5111e949766749b5234926ea710eb/.tmp/info/52b1dc7ee44945399bdf19e24773f306 is 45, key is default/info:d/1700357180203/Put/seqid=0 2023-11-19T01:29:46,269 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/namespace/f8f5111e949766749b5234926ea710eb/.tmp/info/52b1dc7ee44945399bdf19e24773f306 2023-11-19T01:29:46,272 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740/.tmp/info/dca0409bd27f47f0b3f2ae4e37ab26cd is 159, key is ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40./info:regioninfo/1700357322702/Put/seqid=0 2023-11-19T01:29:46,276 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/namespace/f8f5111e949766749b5234926ea710eb/.tmp/info/52b1dc7ee44945399bdf19e24773f306 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/namespace/f8f5111e949766749b5234926ea710eb/info/52b1dc7ee44945399bdf19e24773f306 2023-11-19T01:29:46,276 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.30 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740/.tmp/info/dca0409bd27f47f0b3f2ae4e37ab26cd 2023-11-19T01:29:46,281 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/namespace/f8f5111e949766749b5234926ea710eb/info/52b1dc7ee44945399bdf19e24773f306, entries=7, sequenceid=11, filesize=5.1 K 2023-11-19T01:29:46,282 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~249 B/249, heapSize ~1.01 KB/1032, currentSize=0 B/0 for f8f5111e949766749b5234926ea710eb in 29ms, sequenceid=11, compaction requested=false 2023-11-19T01:29:46,291 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/namespace/f8f5111e949766749b5234926ea710eb/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2023-11-19T01:29:46,291 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:29:46,292 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb. 2023-11-19T01:29:46,292 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for f8f5111e949766749b5234926ea710eb: 2023-11-19T01:29:46,292 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1700357179602.f8f5111e949766749b5234926ea710eb. 2023-11-19T01:29:46,292 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing 322466352cac3ade468ab2ce8e706acc, disabling compactions & flushes 2023-11-19T01:29:46,292 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:29:46,292 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:29:46,292 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. after waiting 0 ms 2023-11-19T01:29:46,292 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:29:46,292 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing 322466352cac3ade468ab2ce8e706acc 2/2 column families, dataSize=1.46 KB heapSize=2.61 KB 2023-11-19T01:29:46,295 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/4718d1227a204ce2b9778a63cadd9ee1 is 171, key is trslm:hdfs://localhost:42739/backupUT\x00ns2:test-17003571846791/meta:log-roll-map/1700357385335/Put/seqid=0 2023-11-19T01:29:46,299 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740/.tmp/rep_barrier/629a82554adc4649a182b879abfd3a2e is 107, key is ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd./rep_barrier:/1700357208705/DeleteFamily/seqid=0 2023-11-19T01:29:46,303 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.13 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740/.tmp/rep_barrier/629a82554adc4649a182b879abfd3a2e 2023-11-19T01:29:46,318 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740/.tmp/table/02cee18f871f4b9bb87212188894987a is 101, key is ns2:test-17003571846791,,1700357187404.dd0bac2484821b3c9f7478186b974ffd./table:/1700357208705/DeleteFamily/seqid=0 2023-11-19T01:29:46,321 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.56 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740/.tmp/table/02cee18f871f4b9bb87212188894987a 2023-11-19T01:29:46,327 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740/.tmp/info/dca0409bd27f47f0b3f2ae4e37ab26cd as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740/info/dca0409bd27f47f0b3f2ae4e37ab26cd 2023-11-19T01:29:46,331 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740/info/dca0409bd27f47f0b3f2ae4e37ab26cd, entries=94, sequenceid=183, filesize=16.4 K 2023-11-19T01:29:46,331 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740/.tmp/rep_barrier/629a82554adc4649a182b879abfd3a2e as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740/rep_barrier/629a82554adc4649a182b879abfd3a2e 2023-11-19T01:29:46,335 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740/rep_barrier/629a82554adc4649a182b879abfd3a2e, entries=12, sequenceid=183, filesize=6.5 K 2023-11-19T01:29:46,336 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740/.tmp/table/02cee18f871f4b9bb87212188894987a as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740/table/02cee18f871f4b9bb87212188894987a 2023-11-19T01:29:46,341 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740/table/02cee18f871f4b9bb87212188894987a, entries=32, sequenceid=183, filesize=7.3 K 2023-11-19T01:29:46,342 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3022): Finished flush of dataSize ~38.00 KB/38910, heapSize ~66.25 KB/67840, currentSize=0 B/0 for 1588230740 in 98ms, sequenceid=183, compaction requested=false 2023-11-19T01:29:46,357 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/hbase/meta/1588230740/recovered.edits/186.seqid, newMaxSeqId=186, maxSeqId=1 2023-11-19T01:29:46,358 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:29:46,358 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2023-11-19T01:29:46,359 INFO [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1909): Closed hbase:meta,,1.1588230740 2023-11-19T01:29:46,359 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1622): Region close journal for 1588230740: 2023-11-19T01:29:46,359 DEBUG [RS_CLOSE_META-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2023-11-19T01:29:46,444 DEBUG [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1609): Waiting on 322466352cac3ade468ab2ce8e706acc, a169e45a4f1d98e65b6a1dada440fc40, b696d85cf645e95a934c0b80f486a6f6, e7af7e5272e1026697882da645f6c4df 2023-11-19T01:29:46,518 INFO [regionserver/jenkins-hbase6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2023-11-19T01:29:46,518 INFO [regionserver/jenkins-hbase6:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2023-11-19T01:29:46,644 DEBUG [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1609): Waiting on 322466352cac3ade468ab2ce8e706acc, a169e45a4f1d98e65b6a1dada440fc40, b696d85cf645e95a934c0b80f486a6f6, e7af7e5272e1026697882da645f6c4df 2023-11-19T01:29:46,700 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=717 B at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/4718d1227a204ce2b9778a63cadd9ee1 2023-11-19T01:29:46,709 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/620fe8e55d8644f1963a890afe88c5fb is 524, key is session:backup_1700357371502/session:context/1700357386163/Put/seqid=0 2023-11-19T01:29:46,845 DEBUG [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1609): Waiting on 322466352cac3ade468ab2ce8e706acc, a169e45a4f1d98e65b6a1dada440fc40, b696d85cf645e95a934c0b80f486a6f6, e7af7e5272e1026697882da645f6c4df 2023-11-19T01:29:47,045 DEBUG [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1609): Waiting on 322466352cac3ade468ab2ce8e706acc, a169e45a4f1d98e65b6a1dada440fc40, b696d85cf645e95a934c0b80f486a6f6, e7af7e5272e1026697882da645f6c4df 2023-11-19T01:29:47,112 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=775 B at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/620fe8e55d8644f1963a890afe88c5fb 2023-11-19T01:29:47,118 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/meta/4718d1227a204ce2b9778a63cadd9ee1 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/4718d1227a204ce2b9778a63cadd9ee1 2023-11-19T01:29:47,124 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/meta/4718d1227a204ce2b9778a63cadd9ee1, entries=6, sequenceid=148, filesize=5.8 K 2023-11-19T01:29:47,125 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/.tmp/session/620fe8e55d8644f1963a890afe88c5fb as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/620fe8e55d8644f1963a890afe88c5fb 2023-11-19T01:29:47,130 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/session/620fe8e55d8644f1963a890afe88c5fb, entries=2, sequenceid=148, filesize=5.5 K 2023-11-19T01:29:47,131 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~1.46 KB/1492, heapSize ~2.58 KB/2640, currentSize=0 B/0 for 322466352cac3ade468ab2ce8e706acc in 839ms, sequenceid=148, compaction requested=true 2023-11-19T01:29:47,155 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system/322466352cac3ade468ab2ce8e706acc/recovered.edits/151.seqid, newMaxSeqId=151, maxSeqId=1 2023-11-19T01:29:47,156 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:29:47,157 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:29:47,157 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for 322466352cac3ade468ab2ce8e706acc: 2023-11-19T01:29:47,157 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system,,1700357193076.322466352cac3ade468ab2ce8e706acc. 2023-11-19T01:29:47,157 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing b696d85cf645e95a934c0b80f486a6f6, disabling compactions & flushes 2023-11-19T01:29:47,157 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6. 2023-11-19T01:29:47,157 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6. 2023-11-19T01:29:47,157 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6. after waiting 0 ms 2023-11-19T01:29:47,157 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6. 2023-11-19T01:29:47,164 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/default/test-1700357184679/b696d85cf645e95a934c0b80f486a6f6/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2023-11-19T01:29:47,165 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:29:47,165 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6. 2023-11-19T01:29:47,165 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for b696d85cf645e95a934c0b80f486a6f6: 2023-11-19T01:29:47,166 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test-1700357184679,,1700357266734.b696d85cf645e95a934c0b80f486a6f6. 2023-11-19T01:29:47,167 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing e7af7e5272e1026697882da645f6c4df, disabling compactions & flushes 2023-11-19T01:29:47,168 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df. 2023-11-19T01:29:47,168 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df. 2023-11-19T01:29:47,168 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df. after waiting 0 ms 2023-11-19T01:29:47,168 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df. 2023-11-19T01:29:47,168 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2819): Flushing e7af7e5272e1026697882da645f6c4df 2/2 column families, dataSize=5.66 KB heapSize=8.50 KB 2023-11-19T01:29:47,180 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system_bulk/e7af7e5272e1026697882da645f6c4df/.tmp/meta/23513b58b27442e29e4c3eb03919d3d8 is 323, key is bulk:ns2:test-17003571846791:4ef412d65669b2fe1684b7fdf6ee1f7d:6157b1620f8c4c3281188261bf9ca9af_SeqId_4_/meta:path/1700357213775/Put/seqid=0 2023-11-19T01:29:47,245 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1579): Waiting on 2 regions to close 2023-11-19T01:29:47,245 DEBUG [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1583): Online Regions={e7af7e5272e1026697882da645f6c4df=backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df., a169e45a4f1d98e65b6a1dada440fc40=ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40.} 2023-11-19T01:29:47,245 DEBUG [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1609): Waiting on a169e45a4f1d98e65b6a1dada440fc40, e7af7e5272e1026697882da645f6c4df 2023-11-19T01:29:47,446 DEBUG [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1609): Waiting on a169e45a4f1d98e65b6a1dada440fc40, e7af7e5272e1026697882da645f6c4df 2023-11-19T01:29:47,585 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.66 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system_bulk/e7af7e5272e1026697882da645f6c4df/.tmp/meta/23513b58b27442e29e4c3eb03919d3d8 2023-11-19T01:29:47,591 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system_bulk/e7af7e5272e1026697882da645f6c4df/.tmp/meta/23513b58b27442e29e4c3eb03919d3d8 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system_bulk/e7af7e5272e1026697882da645f6c4df/meta/23513b58b27442e29e4c3eb03919d3d8 2023-11-19T01:29:47,598 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system_bulk/e7af7e5272e1026697882da645f6c4df/meta/23513b58b27442e29e4c3eb03919d3d8, entries=16, sequenceid=12, filesize=8.1 K 2023-11-19T01:29:47,599 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3022): Finished flush of dataSize ~5.66 KB/5800, heapSize ~8.23 KB/8432, currentSize=0 B/0 for e7af7e5272e1026697882da645f6c4df in 430ms, sequenceid=12, compaction requested=false 2023-11-19T01:29:47,605 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/backup/system_bulk/e7af7e5272e1026697882da645f6c4df/recovered.edits/15.seqid, newMaxSeqId=15, maxSeqId=1 2023-11-19T01:29:47,605 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:29:47,606 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df. 2023-11-19T01:29:47,606 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for e7af7e5272e1026697882da645f6c4df: 2023-11-19T01:29:47,606 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed backup:system_bulk,,1700357194205.e7af7e5272e1026697882da645f6c4df. 2023-11-19T01:29:47,606 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1668): Closing a169e45a4f1d98e65b6a1dada440fc40, disabling compactions & flushes 2023-11-19T01:29:47,606 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1690): Closing region ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40. 2023-11-19T01:29:47,606 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1711): Waiting without time limit for close lock on ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40. 2023-11-19T01:29:47,606 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1778): Acquired close lock on ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40. after waiting 0 ms 2023-11-19T01:29:47,606 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1788): Updates disabled for region ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40. 2023-11-19T01:29:47,609 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/data/ns2/test-17003571846791/a169e45a4f1d98e65b6a1dada440fc40/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2023-11-19T01:29:47,609 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.backup.BackupObserver 2023-11-19T01:29:47,610 INFO [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1909): Closed ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40. 2023-11-19T01:29:47,610 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1622): Region close journal for a169e45a4f1d98e65b6a1dada440fc40: 2023-11-19T01:29:47,610 DEBUG [RS_CLOSE_REGION-regionserver/jenkins-hbase6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed ns2:test-17003571846791,,1700357321544.a169e45a4f1d98e65b6a1dada440fc40. 2023-11-19T01:29:47,646 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase6.apache.org,44997,1700357176784; all regions closed. 2023-11-19T01:29:47,657 DEBUG [RS:0;jenkins-hbase6:44997 {}] wal.AbstractFSWAL(1062): Moved 10 WAL file(s) to /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/oldWALs 2023-11-19T01:29:47,657 INFO [RS:0;jenkins-hbase6:44997 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784.meta:.meta(num 1700357375675) 2023-11-19T01:29:47,665 DEBUG [RS:0;jenkins-hbase6:44997 {}] wal.AbstractFSWAL(1062): Moved 7 WAL file(s) to /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/oldWALs 2023-11-19T01:29:47,665 INFO [RS:0;jenkins-hbase6:44997 {}] wal.AbstractFSWAL(1065): Closed WAL: AsyncFSWAL jenkins-hbase6.apache.org%2C44997%2C1700357176784:(num 1700357375658) 2023-11-19T01:29:47,665 DEBUG [RS:0;jenkins-hbase6:44997 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:47,665 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.LeaseManager(133): Closed leases 2023-11-19T01:29:47,665 INFO [RS:0;jenkins-hbase6:44997 {}] hbase.ChoreService(370): Chore service for: regionserver/jenkins-hbase6:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2023-11-19T01:29:47,666 INFO [regionserver/jenkins-hbase6:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-19T01:29:47,667 INFO [RS:0;jenkins-hbase6:44997 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.2.20:44997 2023-11-19T01:29:47,676 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [jenkins-hbase6.apache.org,44997,1700357176784] 2023-11-19T01:29:47,676 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing jenkins-hbase6.apache.org,44997,1700357176784; numProcessing=1 2023-11-19T01:29:47,678 INFO [RegionServerTracker-0 {}] master.ServerManager(561): Cluster shutdown set; jenkins-hbase6.apache.org,44997,1700357176784 expired; onlineServers=0 2023-11-19T01:29:47,678 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2517): ***** STOPPING region server 'jenkins-hbase6.apache.org,44889,1700357173572' ***** 2023-11-19T01:29:47,678 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2531): STOPPED: Cluster shutdown set; onlineServer=0 2023-11-19T01:29:47,679 DEBUG [M:0;jenkins-hbase6:44889 {}] ipc.AbstractRpcClient(197): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18194831, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=false, bind address=jenkins-hbase6.apache.org/172.31.2.20:0 2023-11-19T01:29:47,679 INFO [M:0;jenkins-hbase6:44889 {}] regionserver.HRegionServer(1172): Stopping infoServer 2023-11-19T01:29:47,724 INFO [M:0;jenkins-hbase6:44889 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.w.WebAppContext@38b9796f{master,/,null,STOPPED}{file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/master} 2023-11-19T01:29:47,724 INFO [M:0;jenkins-hbase6:44889 {}] server.AbstractConnector(383): Stopped ServerConnector@7750fcaa{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-19T01:29:47,724 INFO [M:0;jenkins-hbase6:44889 {}] session.HouseKeeper(149): node0 Stopped scavenging 2023-11-19T01:29:47,725 INFO [M:0;jenkins-hbase6:44889 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@f63ec6a{static,/static,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/classes/hbase-webapps/static/,STOPPED} 2023-11-19T01:29:47,725 INFO [M:0;jenkins-hbase6:44889 {}] handler.ContextHandler(1159): Stopped o.a.h.t.o.e.j.s.ServletContextHandler@2b034ac3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/hadoop.log.dir/,STOPPED} 2023-11-19T01:29:47,726 INFO [M:0;jenkins-hbase6:44889 {}] regionserver.HRegionServer(1213): stopping server jenkins-hbase6.apache.org,44889,1700357173572 2023-11-19T01:29:47,726 INFO [M:0;jenkins-hbase6:44889 {}] regionserver.HRegionServer(1239): stopping server jenkins-hbase6.apache.org,44889,1700357173572; all regions closed. 2023-11-19T01:29:47,726 DEBUG [M:0;jenkins-hbase6:44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:47,726 INFO [M:0;jenkins-hbase6:44889 {}] master.HMaster(1666): Stopping master jetty server 2023-11-19T01:29:47,726 INFO [M:0;jenkins-hbase6:44889 {}] server.AbstractConnector(383): Stopped ServerConnector@4b97ab40{HTTP/1.1, (http/1.1)}{0.0.0.0:0} 2023-11-19T01:29:47,727 DEBUG [M:0;jenkins-hbase6:44889 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2023-11-19T01:29:47,727 INFO [M:0;jenkins-hbase6:44889 {}] master.BackupLogCleaner(171): Stopping BackupLogCleaner 2023-11-19T01:29:47,727 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2023-11-19T01:29:47,727 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.small.0-1700357178363 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.small.0-1700357178363,5,FailOnTimeoutGroup] 2023-11-19T01:29:47,727 DEBUG [master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.large.0-1700357178363 {}] cleaner.HFileCleaner(306): Exit Thread[master/jenkins-hbase6:0:becomeActiveMaster-HFileCleaner.large.0-1700357178363,5,FailOnTimeoutGroup] 2023-11-19T01:29:47,727 DEBUG [M:0;jenkins-hbase6:44889 {}] ipc.AbstractRpcClient(513): Stopping rpc client 2023-11-19T01:29:47,727 DEBUG [M:0;jenkins-hbase6:44889 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2023-11-19T01:29:47,727 INFO [M:0;jenkins-hbase6:44889 {}] hbase.ChoreService(370): Chore service for: master/jenkins-hbase6:0 had [] on shutdown 2023-11-19T01:29:47,727 DEBUG [M:0;jenkins-hbase6:44889 {}] master.HMaster(1689): Stopping service threads 2023-11-19T01:29:47,727 INFO [M:0;jenkins-hbase6:44889 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2023-11-19T01:29:47,728 ERROR [M:0;jenkins-hbase6:44889 {}] procedure2.ProcedureExecutor(654): ThreadGroup java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] contains running threads; null: See STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-12,5,PEWorkerGroup] 2023-11-19T01:29:47,728 INFO [M:0;jenkins-hbase6:44889 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2023-11-19T01:29:47,728 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2023-11-19T01:29:47,729 WARN [M:0;jenkins-hbase6:44889 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2023-11-19T01:29:47,729 INFO [M:0;jenkins-hbase6:44889 {}] assignment.AssignmentManager(383): Stopping assignment manager 2023-11-19T01:29:47,729 INFO [M:0;jenkins-hbase6:44889 {}] region.MasterRegion(178): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2023-11-19T01:29:47,729 DEBUG [M:0;jenkins-hbase6:44889 {}] regionserver.HRegion(1668): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2023-11-19T01:29:47,740 INFO [M:0;jenkins-hbase6:44889 {}] regionserver.HRegion(1690): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:29:47,740 DEBUG [M:0;jenkins-hbase6:44889 {}] regionserver.HRegion(1711): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:29:47,741 DEBUG [M:0;jenkins-hbase6:44889 {}] regionserver.HRegion(1778): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2023-11-19T01:29:47,741 DEBUG [M:0;jenkins-hbase6:44889 {}] regionserver.HRegion(1788): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:29:47,741 INFO [M:0;jenkins-hbase6:44889 {}] regionserver.HRegion(2819): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=621.34 KB heapSize=755.65 KB 2023-11-19T01:29:47,753 DEBUG [M:0;jenkins-hbase6:44889 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/48b55f83d0ea4d4e82bccb49917389ac is 88, key is hbase:meta,,1/info:sn/1700357178833/Put/seqid=0 2023-11-19T01:29:47,757 INFO [M:0;jenkins-hbase6:44889 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=530 B at sequenceid=1608 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/48b55f83d0ea4d4e82bccb49917389ac 2023-11-19T01:29:47,776 INFO [RS:0;jenkins-hbase6:44997 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase6.apache.org,44997,1700357176784; zookeeper connection closed. 2023-11-19T01:29:47,776 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@ce141b6 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@ce141b6 2023-11-19T01:29:47,777 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2023-11-19T01:29:47,782 DEBUG [M:0;jenkins-hbase6:44889 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3eb244ac39ec4ad89d9a1caf73ce6f5b is 986, key is \x00\x00\x00\x00\x00\x00\x00\x1D/proc:d/1700357195398/Put/seqid=0 2023-11-19T01:29:48,186 INFO [M:0;jenkins-hbase6:44889 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=620.75 KB at sequenceid=1608 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3eb244ac39ec4ad89d9a1caf73ce6f5b 2023-11-19T01:29:48,190 INFO [M:0;jenkins-hbase6:44889 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3eb244ac39ec4ad89d9a1caf73ce6f5b 2023-11-19T01:29:48,203 DEBUG [M:0;jenkins-hbase6:44889 {}] hfile.HFileWriterImpl(813): Len of the biggest cell in hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/203154933a3c47ba94ba4ad42d4f78e5 is 82, key is jenkins-hbase6.apache.org,44997,1700357176784/rs:state/1700357178417/Put/seqid=0 2023-11-19T01:29:48,607 INFO [M:0;jenkins-hbase6:44889 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=1608 (bloomFilter=true), to=hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/203154933a3c47ba94ba4ad42d4f78e5 2023-11-19T01:29:48,612 DEBUG [M:0;jenkins-hbase6:44889 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/48b55f83d0ea4d4e82bccb49917389ac as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/48b55f83d0ea4d4e82bccb49917389ac 2023-11-19T01:29:48,617 INFO [M:0;jenkins-hbase6:44889 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/48b55f83d0ea4d4e82bccb49917389ac, entries=8, sequenceid=1608, filesize=5.5 K 2023-11-19T01:29:48,617 DEBUG [M:0;jenkins-hbase6:44889 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3eb244ac39ec4ad89d9a1caf73ce6f5b as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3eb244ac39ec4ad89d9a1caf73ce6f5b 2023-11-19T01:29:48,622 INFO [M:0;jenkins-hbase6:44889 {}] regionserver.StoreFileReader(538): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3eb244ac39ec4ad89d9a1caf73ce6f5b 2023-11-19T01:29:48,622 INFO [M:0;jenkins-hbase6:44889 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3eb244ac39ec4ad89d9a1caf73ce6f5b, entries=173, sequenceid=1608, filesize=39.1 K 2023-11-19T01:29:48,623 DEBUG [M:0;jenkins-hbase6:44889 {}] regionserver.HRegionFileSystem(500): Committing hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/203154933a3c47ba94ba4ad42d4f78e5 as hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/203154933a3c47ba94ba4ad42d4f78e5 2023-11-19T01:29:48,627 INFO [M:0;jenkins-hbase6:44889 {}] regionserver.HStore$StoreFlusherImpl(1984): Added hdfs://localhost:42739/user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/203154933a3c47ba94ba4ad42d4f78e5, entries=1, sequenceid=1608, filesize=5.1 K 2023-11-19T01:29:48,628 INFO [M:0;jenkins-hbase6:44889 {}] regionserver.HRegion(3022): Finished flush of dataSize ~621.34 KB/636251, heapSize ~755.35 KB/773480, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 887ms, sequenceid=1608, compaction requested=false 2023-11-19T01:29:48,632 INFO [M:0;jenkins-hbase6:44889 {}] regionserver.HRegion(1909): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2023-11-19T01:29:48,632 DEBUG [M:0;jenkins-hbase6:44889 {}] regionserver.HRegion(1622): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2023-11-19T01:29:48,640 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(656): complete file /user/jenkins/test-data/4edc650f-2da8-9b58-e039-3bec2e3025af/MasterData/WALs/jenkins-hbase6.apache.org,44889,1700357173572/jenkins-hbase6.apache.org%2C44889%2C1700357173572.1700357177337 not finished, retry = 0 2023-11-19T01:29:48,741 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2023-11-19T01:29:48,741 INFO [M:0;jenkins-hbase6:44889 {}] flush.MasterFlushTableProcedureManager(85): stop: server shutting down. 2023-11-19T01:29:48,741 INFO [M:0;jenkins-hbase6:44889 {}] master.LogRollMasterProcedureManager(73): stop: server shutting down. 2023-11-19T01:29:48,742 INFO [M:0;jenkins-hbase6:44889 {}] ipc.NettyRpcServer(346): Stopping server on /172.31.2.20:44889 2023-11-19T01:29:48,847 INFO [M:0;jenkins-hbase6:44889 {}] regionserver.HRegionServer(1296): Exiting; stopping=jenkins-hbase6.apache.org,44889,1700357173572; zookeeper connection closed. 2023-11-19T01:29:48,855 WARN [Listener at jenkins-hbase6.apache.org/35909 {}] datanode.DirectoryScanner(537): DirectoryScanner: shutdown has been called 2023-11-19T01:29:48,860 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-19T01:29:48,970 WARN [BP-935274450-172.31.2.20-1700357170119 heartbeating to localhost/127.0.0.1:42739 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2023-11-19T01:29:48,970 WARN [BP-935274450-172.31.2.20-1700357170119 heartbeating to localhost/127.0.0.1:42739 {}] datanode.BPServiceActor(862): Ending block pool service for: Block pool BP-935274450-172.31.2.20-1700357170119 (Datanode Uuid 689d6bea-b58f-433c-b8d6-456502b71baf) service to localhost/127.0.0.1:42739 2023-11-19T01:29:48,972 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/cluster_6f56ccd7-1fd7-2f9a-c808-0f86c08a2321/dfs/data/data1/current/BP-935274450-172.31.2.20-1700357170119 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-19T01:29:48,972 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-backup/target/test-data/8c2de40f-2e1a-86c4-ac50-008196feb554/cluster_6f56ccd7-1fd7-2f9a-c808-0f86c08a2321/dfs/data/data2/current/BP-935274450-172.31.2.20-1700357170119 {}] fs.CachingGetSpaceUsed$RefreshThread(211): Thread Interrupted waiting to refresh disk information: sleep interrupted 2023-11-19T01:29:49,021 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:0 2023-11-19T01:29:49,261 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.HBaseTestingUtility(1347): Minicluster is down 2023-11-19T01:29:49,261 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.HBaseTestingUtility(2857): Stopping mini mapreduce cluster... 2023-11-19T01:29:49,272 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:0 2023-11-19T01:29:49,390 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:0 2023-11-19T01:29:49,501 ERROR [Thread[Thread-275,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-19T01:29:49,501 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:0 2023-11-19T01:29:49,608 WARN [ApplicationMaster Launcher {}] amlauncher.ApplicationMasterLauncher$LauncherThread(122): org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher$LauncherThread interrupted. Returning. 2023-11-19T01:29:49,615 ERROR [SchedulerEventDispatcher:Event Processor {}] event.EventDispatcher$EventProcessor(61): Returning, interrupted : java.lang.InterruptedException 2023-11-19T01:29:49,617 ERROR [Thread[Thread-285,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-19T01:29:49,624 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] log.Slf4jLog(67): Stopped HttpServer2$SelectChannelConnectorWithSafeStartup@jenkins-hbase6.apache.org:0 2023-11-19T01:29:49,726 ERROR [Thread[Thread-250,5,FailOnTimeoutGroup] {}] delegation.AbstractDelegationTokenSecretManager$ExpiredTokenRemover(696): ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted 2023-11-19T01:29:49,726 INFO [Listener at jenkins-hbase6.apache.org/35909 {}] hbase.HBaseTestingUtility(2860): Mini mapreduce cluster stopped